def extract( pkcs12Filepath, password, privateKeyFilepath, certificateFilepath ): """Extracts the key and certificate""" try: import OpenSSL.crypto except ImportError: raise estreamer.EncoreException( definitions.STRING_PYOPENSSL_MISSING.format( pkcs12Filepath, privateKeyFilepath, certificateFilepath )) with open( pkcs12Filepath, 'rb' ) as pkcs12File: data = pkcs12File.read() try: pkcs12 = OpenSSL.crypto.load_pkcs12( data, password ) except OpenSSL.crypto.Error: raise estreamer.EncoreException( 'Unable to process pkcs12 file. Possibly a password problem') certificate = pkcs12.get_certificate() privateKey = pkcs12.get_privatekey() # Where type is FILETYPE_PEM or FILETYPE_ASN1 (for DER). cryptoType = OpenSSL.crypto.FILETYPE_PEM with open( privateKeyFilepath, 'wb+' ) as privateKeyFile: privateKeyFile.write( OpenSSL.crypto.dump_privatekey( cryptoType, privateKey ) ) with open( certificateFilepath, 'wb+' ) as certificateFile: certificateFile.write( OpenSSL.crypto.dump_certificate( cryptoType, certificate ) )
def connect(self): """ Opens a secure connection to the remote host """ host = self.settings.host port = self.settings.port self.pkcs12 = estreamer.Crypto.create(settings=self.settings) self.logger.info('Connecting to {0}:{1}'.format(host, port)) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Default TLS tlsVersion = ssl.PROTOCOL_TLSv1 if self.settings.tlsVersion == 1.2: if hasattr(ssl, 'PROTOCOL_TLSv1_2'): tlsVersion = ssl.PROTOCOL_TLSv1_2 self.logger.info('Using TLS v1.2') else: self.logger.warning( 'PROTOCOL_TLSv1_2 not found. Using TLS v1.0') else: self.logger.info('Using TLS v1.0') self.socket = ssl.wrap_socket(sock, keyfile=self.pkcs12.privateKeyFilepath, certfile=self.pkcs12.certificateFilepath, do_handshake_on_connect=True, ssl_version=tlsVersion) try: self.socket.settimeout(self.settings.connectTimeout) self.socket.connect((host, port)) except socket.timeout: raise estreamer.TimeoutException( estreamer.definitions.STRING_CONNECTION_COULD_NOT_CONNECT) except socket.gaierror as gex: # Convert to a nicer exception raise estreamer.EncoreException( 'socket.gaierror ({0})'.format(gex)) except ssl.SSLError as sslex: # Convert to a nicer exception raise estreamer.EncoreException( estreamer.definitions.STRING_CONNECTION_SSL_ERROR.format( sslex, self.pkcs12.privateKeyFilepath, self.pkcs12.certificateFilepath)) # We're setting the socket to be blocking but with a short timeout self.socket.settimeout(self.settings.responseTimeout)
def __init__( self, privateKeyFilepath, certificateFilepath ): if not os.path.isfile( privateKeyFilepath ): raise estreamer.EncoreException( 'privateKeyFilepath: {0} does not exist or is not a file'.format( privateKeyFilepath )) if not os.path.isfile( certificateFilepath ): raise estreamer.EncoreException( 'certificateFilepath: {0} does not exist or is not a file'.format( certificateFilepath )) self.privateKeyFilepath = privateKeyFilepath self.certificateFilepath = certificateFilepath
def response(self): """Returns the next response from the wire""" self.logger.log(logging.TRACE, 'self.__read(8)') dataBuffer = self.__read(8) (version, messageType, length) = struct.unpack('>HHL', dataBuffer) message = { 'version': version, 'messageType': messageType, 'length': length } if self.logger.isEnabledFor(logging.TRACE): self.logger.log(logging.TRACE, 'header: {0}'.format(binascii.hexlify(dataBuffer))) self.logger.log(logging.TRACE, message) if version != 1: raise estreamer.EncoreException( estreamer.definitions.STRING_CONNECTION_INVALID_HEADER.format( version, message)) if version == 1 and messageType != 0: self.lastReceiveTime = datetime.datetime.now().now() if not self.firstReceiveTime: self.firstReceiveTime = self.lastReceiveTime if version == 1 and length > 0: if self.logger.isEnabledFor(logging.TRACE): self.logger.log(logging.TRACE, 'self.__read({0})'.format(length)) message['data'] = self.__read(length) if self.logger.isEnabledFor(logging.TRACE): self.logger.log( logging.TRACE, 'data: {0}'.format(binascii.hexlify(message['data']))) actualLength = len(message['data']) if length != actualLength: raise estreamer.EncoreException( 'Expected length {0} but got {1}'.format( length, actualLength)) if self.logger.isEnabledFor(logging.TRACE): self.logger.log(logging.TRACE, str(message)) return message
def _shouldParse( record, settings ): """ Decides if a record should be decoded. This will mostly look at whether or not we need to write the record on the basis of its recordType, however, metadata records should always be decoded since we need to write them into our cache. The main reason for this check is performance. Parsing is expensive and we can save significant time here for certain records. we also do a check after this too to see if the user wants us to decode. """ recordTypeId = 0 if 'recordType' in record: recordTypeId = record['recordType'] if recordTypeId == 0: return False if recordTypeId not in definitions.RECORDS: raise estreamer.EncoreException( 'Record type {0} is unknown : not in scope'.format( recordTypeId )) result = False if recordTypeId in definitions.TYPES['CONNECTION']: if settings.writeConnections: result = True elif recordTypeId in definitions.TYPES['RNA']: if settings.writeRna: result = True elif recordTypeId in definitions.TYPES['RUA']: if settings.writeRua: result = True elif recordTypeId in definitions.TYPES['METADATA']: result = True elif recordTypeId in definitions.TYPES['PACKET']: if settings.writePackets: result = True elif recordTypeId in definitions.TYPES['INTRUSION']: if settings.writeIntrusion: result = True elif recordTypeId in definitions.TYPES['FILE_MALWARE'] or \ recordTypeId in definitions.TYPES['CORRELATION'] or \ recordTypeId in definitions.TYPES['EVENT']: if settings.writeCore: result = True # Regardless of above if recordTypeId in settings.writeRecordTypes: result = True if recordTypeId in settings.writeExcludeRecordTypes: result = False return result
def _requestStreamingInformation(self, responseMessage): offset = 0 serviceId = 0 gotService = False while offset < len(responseMessage['data']): weeBuffer = responseMessage['data'][offset:offset + 8] (serviceId, length) = struct.unpack('>LL', weeBuffer) if serviceId == definitions.MESSAGE_STREAMING_INFORMATION_REQUEST_SERVICE_ID: gotService = True break offset = offset + 8 + length if not gotService: raise estreamer.EncoreException('No StreamingInformation service') serviceMessage = estreamer.message.StreamingRequestMessage( self.settings) messageHex = binascii.hexlify(serviceMessage.getWireData()) self.logger.info('StreamingRequestMessage: {0}'.format(messageHex)) self.connection.request(serviceMessage)
def create(self): """Creates the pid file - raises an exception if it already exists""" if self.exists(): raise estreamer.EncoreException('PID file already exists') with open(self.filepath, 'w') as pidFile: pidFile.write(str(os.getpid()))
def __init__(self, directory, threshold, rotate, filename, uri, scpKeyFilepath, encoding='utf-8', deleteOnTransfer=True): super(ScpStream, self).__init__(directory, threshold, rotate, filename, encoding) self.logger = estreamer.crossprocesslogging.getLogger(__name__) self.uri = uri self.scpKeyFilepath = scpKeyFilepath self.port = 22 self.deleteOnTransfer = deleteOnTransfer self.transfer = None self._setHandler() if self.uri.port: if convert.isUint16(self.uri.port): self.port = self.uri.port else: raise estreamer.EncoreException( definitions.STRING_INVALID_PORT.format(self.uri.port))
def create( settings = None, password = None, pkcs12Filepath = None ): """ Creates a Crypto instance from a local public / private key pair but throws an exception if the pair does not exist """ if settings is not None: if password is None: return Crypto( settings.privateKeyFilepath(), settings.publicKeyFilepath()) else: if not os.path.isfile( settings.pkcs12Filepath ): raise estreamer.EncoreException( 'pkcs12Filepath: {0} does not exist or is not a file'.format( settings.pkcs12Filepath )) certificateFilepath = settings.publicKeyFilepath() privateKeyFilepath = settings.privateKeyFilepath() Crypto.extract( settings.pkcs12Filepath, password, privateKeyFilepath, certificateFilepath ) return Crypto( privateKeyFilepath, certificateFilepath ) elif pkcs12Filepath is not None: # This should only be used for unit testing if not os.path.isfile( pkcs12Filepath ): raise estreamer.EncoreException( 'pkcs12Filepath: {0} does not exist or is not a file'.format( pkcs12Filepath )) certificateFilepath = pkcs12Filepath + '.cert' privateKeyFilepath = pkcs12Filepath + '.key' Crypto.extract( pkcs12Filepath, password, privateKeyFilepath, certificateFilepath ) return Crypto( privateKeyFilepath, certificateFilepath )
def _radixAsFormat( radix ): if radix == 2: return '0>08b' elif radix == 8: return '0>04o' elif radix == 16: return '0>02x' raise estreamer.EncoreException('Unknown formatter radix')
def _handleControlCommand(self, command): if command == 'stop': self.pipe.send('ok') self.logger.info('Stop message received') self.stop() elif command == 'status': self.pipe.send(self.status()) else: raise estreamer.EncoreException( 'Unknown command: {0}'.format(command))
def create( filepath ): """Creates a new settings object and initialises logging""" if not os.path.isfile( filepath ): raise estreamer.EncoreException( 'Settings file: {0} does not exist or is not a file'.format( filepath )) with open( filepath, 'r' ) as configFile: try: config = json.load( configFile ) settings = Settings( config ) return settings except ValueError: raise estreamer.ParsingException('Invalid JSON in settings file')
def getLogger(name=None): """Gets the multi-process aware logger for this process""" if IsMultiProcess: if __serverQueueInstance is None: msg = 'Process logging not initialised. Call ' + \ 'estreamer.crossprocesslogging.init( queue, level )' if Print: print(msg) raise estreamer.EncoreException(msg) else: return Client(__serverQueueInstance, name, __logLevelId) else: return __logging.getLogger(name)
def adapters( self ): if self._adapters is None: self._adapters = [] for settingsOutputter in self.outputters: try: adapter = __import__( 'estreamer.adapters.{0}'.format( settingsOutputter.adapter ), fromlist = ['estreamer.adapters'] ) self._adapters.append( adapter ) except ImportError: raise estreamer.EncoreException( 'Unrecognised adapter: {0}'.format( settingsOutputter.adapter )) return self._adapters
def __packetData(data): payload = '' packet = estreamer.common.Packet.createFromHex(data) if PACKET_ENCODING == 'ascii': payload = packet.getPayloadAsAscii() elif PACKET_ENCODING == 'utf8': payload = packet.getPayloadAsUtf8() elif PACKET_ENCODING == 'hex': payload = packet.getPayloadAsHex() else: raise estreamer.EncoreException('Unknown packet encoding') return payload[0:PACKET_LENGTH_MAX]
def execute(self): """Executes a simple diagnostic run""" if self.settings.reprocessPkcs12: self.logger.info('Clearing certificate and key') estreamer.Crypto.create(settings=self.settings).clean() self.logger.info('Check certificate') try: estreamer.Crypto.create(settings=self.settings) except estreamer.EncoreException: self.logger.info('PKCS12 file needs processing') # Wait half a second for logs to be written otherwise password prompt gets lost time.sleep(definitions.TIME_PAUSE) try: password = getpass.getpass( prompt=definitions.STRING_PASSWORD_PROMPT) except EOFError: raise estreamer.EncoreException( definitions.STRING_PASSWORD_STDIN_EOF) try: estreamer.Crypto.create(settings=self.settings, password=password) except estreamer.EncoreException: raise self.logger.info('Creating connection') connection = estreamer.Connection(self.settings) connection.connect() self.logger.info('Creating request message') timestamp = definitions.TIMESTAMP_NOW flags = self.settings.requestFlags() message = estreamer.message.EventStreamRequestMessage(timestamp, flags) self.logger.info('Request message={0}'.format( binascii.hexlify(message.getWireData()))) try: self.logger.info('Sending request message') connection.request(message) self.logger.info('Receiving response message') response = connection.response() except estreamer.ConnectionClosedException: self.logger.error(definitions.STRING_CONNECTION_CLOSED) raise self.logger.info('Response message={0}'.format( estreamer.adapters.base64.dumps(response))) if response[ 'messageType'] == definitions.MESSAGE_TYPE_STREAMING_INFORMATION: self.logger.info('Streaming info response') elif response[ 'messageType'] == definitions.MESSAGE_TYPE_MESSAGE_BUNDLE: self.logger.info('Bundle') elif response['messageType'] == definitions.MESSAGE_TYPE_NULL: self.logger.info('Null') elif response['messageType'] == definitions.MESSAGE_TYPE_EVENT_DATA: self.logger.info('Parsing response message') record = estreamer.pipeline.parse(response, self.settings) self.logger.info('Response record={0}'.format(record)) elif response['messageType'] == definitions.MESSAGE_TYPE_ERROR: self.logger.error('Error message received: {0}'.format( estreamer.adapters.base64.dumps(message))) else: self.logger.warning('Unknown message received: {0}'.format( estreamer.adapters.base64.dumps(message))) self.logger.info('Connection successful')