def extract( pkcs12Filepath, password, privateKeyFilepath, certificateFilepath ):
"""Extracts the key and certificate"""
try:
import OpenSSL.crypto
except ImportError:
raise estreamer.EncoreException(
definitions.STRING_PYOPENSSL_MISSING.format(
pkcs12Filepath,
privateKeyFilepath,
certificateFilepath ))
with open( pkcs12Filepath, 'rb' ) as pkcs12File:
data = pkcs12File.read()
try:
pkcs12 = OpenSSL.crypto.load_pkcs12( data, password )
except OpenSSL.crypto.Error:
raise estreamer.EncoreException(
'Unable to process pkcs12 file. Possibly a password problem')
certificate = pkcs12.get_certificate()
privateKey = pkcs12.get_privatekey()
# Where type is FILETYPE_PEM or FILETYPE_ASN1 (for DER).
cryptoType = OpenSSL.crypto.FILETYPE_PEM
with open( privateKeyFilepath, 'wb+' ) as privateKeyFile:
privateKeyFile.write( OpenSSL.crypto.dump_privatekey( cryptoType, privateKey ) )
with open( certificateFilepath, 'wb+' ) as certificateFile:
certificateFile.write( OpenSSL.crypto.dump_certificate( cryptoType, certificate ) )
def connect(self):
"""
Opens a secure connection to the remote host
"""
host = self.settings.host
port = self.settings.port
self.pkcs12 = estreamer.Crypto.create(settings=self.settings)
self.logger.info('Connecting to {0}:{1}'.format(host, port))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Default TLS
tlsVersion = ssl.PROTOCOL_TLSv1
if self.settings.tlsVersion == 1.2:
if hasattr(ssl, 'PROTOCOL_TLSv1_2'):
tlsVersion = ssl.PROTOCOL_TLSv1_2
self.logger.info('Using TLS v1.2')
else:
self.logger.warning(
'PROTOCOL_TLSv1_2 not found. Using TLS v1.0')
else:
self.logger.info('Using TLS v1.0')
self.socket = ssl.wrap_socket(sock,
keyfile=self.pkcs12.privateKeyFilepath,
certfile=self.pkcs12.certificateFilepath,
do_handshake_on_connect=True,
ssl_version=tlsVersion)
try:
self.socket.settimeout(self.settings.connectTimeout)
self.socket.connect((host, port))
except socket.timeout:
raise estreamer.TimeoutException(
estreamer.definitions.STRING_CONNECTION_COULD_NOT_CONNECT)
except socket.gaierror as gex:
# Convert to a nicer exception
raise estreamer.EncoreException(
'socket.gaierror ({0})'.format(gex))
except ssl.SSLError as sslex:
# Convert to a nicer exception
raise estreamer.EncoreException(
estreamer.definitions.STRING_CONNECTION_SSL_ERROR.format(
sslex, self.pkcs12.privateKeyFilepath,
self.pkcs12.certificateFilepath))
# We're setting the socket to be blocking but with a short timeout
self.socket.settimeout(self.settings.responseTimeout)
def __init__( self, privateKeyFilepath, certificateFilepath ):
if not os.path.isfile( privateKeyFilepath ):
raise estreamer.EncoreException(
'privateKeyFilepath: {0} does not exist or is not a file'.format(
privateKeyFilepath ))
if not os.path.isfile( certificateFilepath ):
raise estreamer.EncoreException(
'certificateFilepath: {0} does not exist or is not a file'.format(
certificateFilepath ))
self.privateKeyFilepath = privateKeyFilepath
self.certificateFilepath = certificateFilepath
def response(self):
"""Returns the next response from the wire"""
self.logger.log(logging.TRACE, 'self.__read(8)')
dataBuffer = self.__read(8)
(version, messageType, length) = struct.unpack('>HHL', dataBuffer)
message = {
'version': version,
'messageType': messageType,
'length': length
}
if self.logger.isEnabledFor(logging.TRACE):
self.logger.log(logging.TRACE,
'header: {0}'.format(binascii.hexlify(dataBuffer)))
self.logger.log(logging.TRACE, message)
if version != 1:
raise estreamer.EncoreException(
estreamer.definitions.STRING_CONNECTION_INVALID_HEADER.format(
version, message))
if version == 1 and messageType != 0:
self.lastReceiveTime = datetime.datetime.now().now()
if not self.firstReceiveTime:
self.firstReceiveTime = self.lastReceiveTime
if version == 1 and length > 0:
if self.logger.isEnabledFor(logging.TRACE):
self.logger.log(logging.TRACE,
'self.__read({0})'.format(length))
message['data'] = self.__read(length)
if self.logger.isEnabledFor(logging.TRACE):
self.logger.log(
logging.TRACE,
'data: {0}'.format(binascii.hexlify(message['data'])))
actualLength = len(message['data'])
if length != actualLength:
raise estreamer.EncoreException(
'Expected length {0} but got {1}'.format(
length, actualLength))
if self.logger.isEnabledFor(logging.TRACE):
self.logger.log(logging.TRACE, str(message))
return message
def _shouldParse( record, settings ):
"""
Decides if a record should be decoded. This will mostly look at whether or not
we need to write the record on the basis of its recordType, however, metadata
records should always be decoded since we need to write them into our cache.
The main reason for this check is performance. Parsing is expensive and we can
save significant time here for certain records. we also do a check after this
too to see if the user wants us to decode.
"""
recordTypeId = 0
if 'recordType' in record:
recordTypeId = record['recordType']
if recordTypeId == 0:
return False
if recordTypeId not in definitions.RECORDS:
raise estreamer.EncoreException(
'Record type {0} is unknown : not in scope'.format( recordTypeId ))
result = False
if recordTypeId in definitions.TYPES['CONNECTION']:
if settings.writeConnections:
result = True
elif recordTypeId in definitions.TYPES['RNA']:
if settings.writeRna:
result = True
elif recordTypeId in definitions.TYPES['RUA']:
if settings.writeRua:
result = True
elif recordTypeId in definitions.TYPES['METADATA']:
result = True
elif recordTypeId in definitions.TYPES['PACKET']:
if settings.writePackets:
result = True
elif recordTypeId in definitions.TYPES['INTRUSION']:
if settings.writeIntrusion:
result = True
elif recordTypeId in definitions.TYPES['FILE_MALWARE'] or \
recordTypeId in definitions.TYPES['CORRELATION'] or \
recordTypeId in definitions.TYPES['EVENT']:
if settings.writeCore:
result = True
# Regardless of above
if recordTypeId in settings.writeRecordTypes:
result = True
if recordTypeId in settings.writeExcludeRecordTypes:
result = False
return result
def _requestStreamingInformation(self, responseMessage):
offset = 0
serviceId = 0
gotService = False
while offset < len(responseMessage['data']):
weeBuffer = responseMessage['data'][offset:offset + 8]
(serviceId, length) = struct.unpack('>LL', weeBuffer)
if serviceId == definitions.MESSAGE_STREAMING_INFORMATION_REQUEST_SERVICE_ID:
gotService = True
break
offset = offset + 8 + length
if not gotService:
raise estreamer.EncoreException('No StreamingInformation service')
serviceMessage = estreamer.message.StreamingRequestMessage(
self.settings)
messageHex = binascii.hexlify(serviceMessage.getWireData())
self.logger.info('StreamingRequestMessage: {0}'.format(messageHex))
self.connection.request(serviceMessage)
def create(self):
"""Creates the pid file - raises an exception if it already exists"""
if self.exists():
raise estreamer.EncoreException('PID file already exists')
with open(self.filepath, 'w') as pidFile:
pidFile.write(str(os.getpid()))
def __init__(self,
directory,
threshold,
rotate,
filename,
uri,
scpKeyFilepath,
encoding='utf-8',
deleteOnTransfer=True):
super(ScpStream, self).__init__(directory, threshold, rotate, filename,
encoding)
self.logger = estreamer.crossprocesslogging.getLogger(__name__)
self.uri = uri
self.scpKeyFilepath = scpKeyFilepath
self.port = 22
self.deleteOnTransfer = deleteOnTransfer
self.transfer = None
self._setHandler()
if self.uri.port:
if convert.isUint16(self.uri.port):
self.port = self.uri.port
else:
raise estreamer.EncoreException(
definitions.STRING_INVALID_PORT.format(self.uri.port))
def create( settings = None, password = None, pkcs12Filepath = None ):
"""
Creates a Crypto instance from a local public / private key pair but throws
an exception if the pair does not exist
"""
if settings is not None:
if password is None:
return Crypto(
settings.privateKeyFilepath(),
settings.publicKeyFilepath())
else:
if not os.path.isfile( settings.pkcs12Filepath ):
raise estreamer.EncoreException(
'pkcs12Filepath: {0} does not exist or is not a file'.format(
settings.pkcs12Filepath ))
certificateFilepath = settings.publicKeyFilepath()
privateKeyFilepath = settings.privateKeyFilepath()
Crypto.extract(
settings.pkcs12Filepath,
password,
privateKeyFilepath,
certificateFilepath )
return Crypto( privateKeyFilepath, certificateFilepath )
elif pkcs12Filepath is not None:
# This should only be used for unit testing
if not os.path.isfile( pkcs12Filepath ):
raise estreamer.EncoreException(
'pkcs12Filepath: {0} does not exist or is not a file'.format(
pkcs12Filepath ))
certificateFilepath = pkcs12Filepath + '.cert'
privateKeyFilepath = pkcs12Filepath + '.key'
Crypto.extract(
pkcs12Filepath,
password,
privateKeyFilepath,
certificateFilepath )
return Crypto( privateKeyFilepath, certificateFilepath )
def _radixAsFormat( radix ):
if radix == 2:
return '0>08b'
elif radix == 8:
return '0>04o'
elif radix == 16:
return '0>02x'
raise estreamer.EncoreException('Unknown formatter radix')
def _handleControlCommand(self, command):
if command == 'stop':
self.pipe.send('ok')
self.logger.info('Stop message received')
self.stop()
elif command == 'status':
self.pipe.send(self.status())
else:
raise estreamer.EncoreException(
'Unknown command: {0}'.format(command))
def create( filepath ):
"""Creates a new settings object and initialises logging"""
if not os.path.isfile( filepath ):
raise estreamer.EncoreException(
'Settings file: {0} does not exist or is not a file'.format(
filepath ))
with open( filepath, 'r' ) as configFile:
try:
config = json.load( configFile )
settings = Settings( config )
return settings
except ValueError:
raise estreamer.ParsingException('Invalid JSON in settings file')
def getLogger(name=None):
"""Gets the multi-process aware logger for this process"""
if IsMultiProcess:
if __serverQueueInstance is None:
msg = 'Process logging not initialised. Call ' + \
'estreamer.crossprocesslogging.init( queue, level )'
if Print:
print(msg)
raise estreamer.EncoreException(msg)
else:
return Client(__serverQueueInstance, name, __logLevelId)
else:
return __logging.getLogger(name)
def adapters( self ):
if self._adapters is None:
self._adapters = []
for settingsOutputter in self.outputters:
try:
adapter = __import__(
'estreamer.adapters.{0}'.format( settingsOutputter.adapter ),
fromlist = ['estreamer.adapters'] )
self._adapters.append( adapter )
except ImportError:
raise estreamer.EncoreException(
'Unrecognised adapter: {0}'.format( settingsOutputter.adapter ))
return self._adapters
def __packetData(data):
payload = ''
packet = estreamer.common.Packet.createFromHex(data)
if PACKET_ENCODING == 'ascii':
payload = packet.getPayloadAsAscii()
elif PACKET_ENCODING == 'utf8':
payload = packet.getPayloadAsUtf8()
elif PACKET_ENCODING == 'hex':
payload = packet.getPayloadAsHex()
else:
raise estreamer.EncoreException('Unknown packet encoding')
return payload[0:PACKET_LENGTH_MAX]
def execute(self):
"""Executes a simple diagnostic run"""
if self.settings.reprocessPkcs12:
self.logger.info('Clearing certificate and key')
estreamer.Crypto.create(settings=self.settings).clean()
self.logger.info('Check certificate')
try:
estreamer.Crypto.create(settings=self.settings)
except estreamer.EncoreException:
self.logger.info('PKCS12 file needs processing')
# Wait half a second for logs to be written otherwise password prompt gets lost
time.sleep(definitions.TIME_PAUSE)
try:
password = getpass.getpass(
prompt=definitions.STRING_PASSWORD_PROMPT)
except EOFError:
raise estreamer.EncoreException(
definitions.STRING_PASSWORD_STDIN_EOF)
try:
estreamer.Crypto.create(settings=self.settings,
password=password)
except estreamer.EncoreException:
raise
self.logger.info('Creating connection')
connection = estreamer.Connection(self.settings)
connection.connect()
self.logger.info('Creating request message')
timestamp = definitions.TIMESTAMP_NOW
flags = self.settings.requestFlags()
message = estreamer.message.EventStreamRequestMessage(timestamp, flags)
self.logger.info('Request message={0}'.format(
binascii.hexlify(message.getWireData())))
try:
self.logger.info('Sending request message')
connection.request(message)
self.logger.info('Receiving response message')
response = connection.response()
except estreamer.ConnectionClosedException:
self.logger.error(definitions.STRING_CONNECTION_CLOSED)
raise
self.logger.info('Response message={0}'.format(
estreamer.adapters.base64.dumps(response)))
if response[
'messageType'] == definitions.MESSAGE_TYPE_STREAMING_INFORMATION:
self.logger.info('Streaming info response')
elif response[
'messageType'] == definitions.MESSAGE_TYPE_MESSAGE_BUNDLE:
self.logger.info('Bundle')
elif response['messageType'] == definitions.MESSAGE_TYPE_NULL:
self.logger.info('Null')
elif response['messageType'] == definitions.MESSAGE_TYPE_EVENT_DATA:
self.logger.info('Parsing response message')
record = estreamer.pipeline.parse(response, self.settings)
self.logger.info('Response record={0}'.format(record))
elif response['messageType'] == definitions.MESSAGE_TYPE_ERROR:
self.logger.error('Error message received: {0}'.format(
estreamer.adapters.base64.dumps(message)))
else:
self.logger.warning('Unknown message received: {0}'.format(
estreamer.adapters.base64.dumps(message)))
self.logger.info('Connection successful')
