def handlerHcp(self, c, messages): for message in messages: if 'hcp.MODULES' in message: moduleUpdateResp = self.moduleManager.request( 'sync', { 'mods': message['hcp.MODULES'], 'aid': c.getAid(), 'tags': c.tags }, timeout=30) if moduleUpdateResp.isSuccess: changes = moduleUpdateResp.data['changes'] tasks = [] for mod in changes['unload']: tasks.append(rSequence().addInt8( Symbols.base.OPERATION, HcpOperations.UNLOAD_MODULE).addInt8( Symbols.hcp.MODULE_ID, mod)) for mod in changes['load']: tasks.append(rSequence().addInt8( Symbols.base.OPERATION, HcpOperations.LOAD_MODULE).addInt8( Symbols.hcp.MODULE_ID, mod[0]).addBuffer( Symbols.base.BINARY, mod[2]).addBuffer(Symbols.base.SIGNATURE, mod[3])) c.sendFrame(HcpModuleId.HCP, tasks) self.log('load %d modules, unload %d modules' % (len(changes['load']), len(changes['unload']))) else: self.log("could not provide module sync: %s" % moduleUpdateResp.error)
def hbs_taskAgent( self, toAgent, task, key, id, expiry = None, investigationId = None ): # Make sure it's a valid agentid a = AgentId( toAgent ) if not a.isValid: return None if not type( task ) is rSequence: return None s = Signing( key ) r = rpcm( isHumanReadable = True, isDebug = True ) tags = Symbols() if investigationId is not None and '' != investigationId: task.addStringA( tags.hbs.INVESTIGATION_ID, investigationId ) toSign = ( rSequence().addSequence( tags.base.HCP_ID, rSequence().addInt8( tags.base.HCP_ID_ORG, a.org ) .addInt8( tags.base.HCP_ID_SUBNET, a.subnet ) .addInt32( tags.base.HCP_ID_UNIQUE, a.unique ) .addInt8( tags.base.HCP_ID_PLATFORM, a.platform ) .addInt8( tags.base.HCP_ID_CONFIG, a.config ) ) .addSequence( tags.hbs.NOTIFICATION, task ) .addInt32( tags.hbs.NOTIFICATION_ID, id ) ) if None != expiry: toSign.addTimestamp( tags.base.EXPIRY, int( expiry ) ) toSign = r.serialise( toSign ) sig = s.sign( toSign ) final = r.serialise( rSequence().addBuffer( tags.base.BINARY, toSign ) .addBuffer( tags.base.SIGNATURE, sig ) ) return self._query( 'hbs.task_agent', { 'task' : final, 'agentid' : str( a ) } )
def handlerHbs(self, c, messages): for i in range(len(messages)): self.processedCounter += 1 if 0 == (self.processedCounter % 1000): self.log('EP_IN %s' % self.processedCounter) for message in messages: # We treat sync messages slightly differently since they need to be actioned # more directly. if 'notification.SYNC' in message: self.log("sync received from %s" % c.getAid()) profileHash = message['notification.SYNC'].get( 'base.HASH', None) profileUpdateResp = self.hbsProfileManager.request( 'sync', { 'hprofile': profileHash, 'aid': c.getAid(), 'tags': c.tags }, timeout=30) if profileUpdateResp.isSuccess and 'changes' in profileUpdateResp.data: profile = profileUpdateResp.data['changes'].get( 'profile', None) if profile is not None: r = rpcm(isHumanReadable=False, isDebug=self.log, isDetailedDeserialize=True) r.setBuffer(profile[0]) realProfile = r.deserialise(isList=True) if realProfile is not None: syncProfile = rSequence().addSequence( Symbols.notification.SYNC, rSequence().addBuffer( Symbols.base.HASH, profile[1].decode('hex')).addList( Symbols.hbs.CONFIGURATIONS, realProfile)) c.sendFrame(HcpModuleId.HBS, (syncProfile, )) self.log("sync profile sent to %s" % c.getAid()) # Transmit the message to the analytics cloud. routing = { 'aid': c.getAid(), 'hostname': c.hostName, 'int_ip': c.int_ip, 'ext_ip': c.ext_ip, 'moduleid': HcpModuleId.HBS, 'event_type': message.keys()[0], 'event_time': message.values()[0].get('base.TIMESTAMP', None), 'event_id': uuid.uuid4(), 'tags': c.tags } invId = message.values()[0].get('hbs.INVESTIGATION_ID', None) if invId is not None: routing['investigation_id'] = invId self.analyticsIntake.shoot('analyze', ((routing, message), ), timeout=600)
def hbs_taskAgent(self, toAgent, task, key, id, expiry=None, investigationId=None): # Make sure it's a valid agentid a = AgentId(toAgent) if not type(task) is rSequence: return None s = Signing(key) r = rpcm(isHumanReadable=True, isDebug=True) tags = Symbols() if investigationId is not None and '' != investigationId: task.addStringA(tags.hbs.INVESTIGATION_ID, investigationId) toSign = (rSequence().addSequence( tags.base.HCP_IDENT, rSequence().addBuffer( tags.base.HCP_SENSOR_ID, (a.sensor_id if a.sensor_id is not None else self.empty_uuid).bytes).addBuffer( tags.base.HCP_ORG_ID, (a.org_id if a.org_id is not None else self.empty_uuid).bytes).addBuffer( tags.base.HCP_INSTALLER_ID, (a.ins_id if a.ins_id is not None else self.empty_uuid).bytes).addInt32( tags.base.HCP_ARCHITECTURE, a.architecture if a.architecture is not None else 0).addInt32( tags.base.HCP_PLATFORM, a.platform if a.platform is not None else 0)).addSequence( tags.hbs.NOTIFICATION, task).addInt32(tags.hbs.NOTIFICATION_ID, id)) if None != expiry: toSign.addTimestamp(tags.base.EXPIRY, int(expiry)) toSign = r.serialise(toSign) sig = s.sign(toSign) final = r.serialise(rSequence().addBuffer( tags.base.BINARY, toSign).addBuffer(tags.base.SIGNATURE, sig)) return self._query('hbs.task_agent', { 'task': final, 'aid': str(a), 'expiry': expiry })
def do_history_dump(self, s): """Dump the full recent history of events on the sensor.""" parser = self.getParser("history_dump", True) arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking(self.tags.notification.HISTORY_DUMP_REQ, rSequence(), arguments)
def do_os_autoruns(self, s): """Generate a new autoruns snapshot.""" parser = self.getParser("os_autoruns", True) arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking(self.tags.notification.OS_AUTORUNS_REQ, rSequence(), arguments)
def do_os_processes(self, s): """Generate a new process snapshot.""" parser = self.getParser("os_processes", True) arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking(self.tags.notification.OS_PROCESSES_REQ, rSequence(), arguments)
def do_os_drivers(self, s): """Get the drivers registered on the host.""" parser = self.getParser("os_drivers", True) arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking(self.tags.notification.OS_DRIVERS_REQ, rSequence(), arguments)
def do_os_services(self, s): """Get the services registered on the host.""" parser = self.getParser("getServices", True) arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking(self.tags.notification.OS_SERVICES_REQ, rSequence(), arguments)
def do_dir_list(self, s): """Get the directory listing.""" parser = self.getParser("dir_list", True) parser.add_argument("rootDir", type=unicode, help="the root directory where to begin the listing from") parser.add_argument( "fileExp", type=unicode, help="a file name expression supporting basic wildcards like * and ?" ) parser.add_argument( "-d", "--depth", dest="depth", required=False, default=0, help="optional maximum depth of the listing, defaults to a single level", ) arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking( self.tags.hbs.NOTIFICATION_DIR_LIST_REQ, rSequence() .addStringW(self.tags.base.FILE_PATH, arguments.fileExp) .addStringW(self.tags.base.DIRECTORY_PATH, arguments.rootDir) .addInt32(self.tags.base.DIRECTORY_LIST_DEPTH, arguments.depth), arguments, )
def do_os_services( self, s ): '''Get the services registered on the host.''' parser = self.getParser( 'getServices', True ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.OS_SERVICES_REQ, rSequence(), arguments )
def do_os_drivers( self, s ): '''Get the drivers registered on the host.''' parser = self.getParser( 'os_drivers', True ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.OS_DRIVERS_REQ, rSequence(), arguments )
def do_os_processes( self, s ): '''Generate a new process snapshot.''' parser = self.getParser( 'os_processes', True ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.OS_PROCESSES_REQ, rSequence(), arguments )
def do_os_autoruns( self, s ): '''Generate a new autoruns snapshot.''' parser = self.getParser( 'os_autoruns', True ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.OS_AUTORUNS_REQ, rSequence(), arguments )
def do_history_dump( self, s ): '''Dump the full recent history of events on the sensor.''' parser = self.getParser( 'history_dump', True ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.HISTORY_DUMP_REQ, rSequence(), arguments )
def do_critical_get( self, s ): '''Show which custom events are critical (other than through the global profile).''' parser = self.getParser( 'critical_get', True ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.GET_CRITICAL_EVENT_REQ, rSequence(), arguments )
def hbs_taskAgent(self, toAgent, task, key, id, expiry=None, investigationId=None): # Make sure it's a valid agentid a = AgentId(toAgent) if not a.isValid: return None if not type(task) is rSequence: return None s = Signing(key) r = rpcm(isHumanReadable=True, isDebug=True) tags = Symbols() if investigationId is not None and '' != investigationId: task.addStringA(tags.hbs.INVESTIGATION_ID, investigationId) toSign = (rSequence().addSequence( tags.base.HCP_ID, rSequence().addInt8(tags.base.HCP_ID_ORG, a.org).addInt8( tags.base.HCP_ID_SUBNET, a.subnet).addInt32(tags.base.HCP_ID_UNIQUE, a.unique).addInt8( tags.base.HCP_ID_PLATFORM, a.platform).addInt8( tags.base.HCP_ID_CONFIG, a.config)).addSequence( tags.hbs.NOTIFICATION, task).addInt32(tags.hbs.NOTIFICATION_ID, id)) if None != expiry: toSign.addTimestamp(tags.base.EXPIRY, int(expiry)) toSign = r.serialise(toSign) sig = s.sign(toSign) final = r.serialise(rSequence().addBuffer( tags.base.BINARY, toSign).addBuffer(tags.base.SIGNATURE, sig)) return self._query('hbs.task_agent', { 'task': final, 'agentid': str(a), 'expiry': expiry })
def do_os_kill_process( self, s ): '''Kill a process on the host.''' parser = self.getParser( 'os_kill_process', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to kill' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.OS_KILL_PROCESS_REQ, rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ), arguments )
def do_mem_strings( self, s ): '''Get the strings from a specific process.''' parser = self.getParser( 'mem_strings', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to get the strings from' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.MEM_STRINGS_REQ, rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ), arguments )
def do_mem_handles( self, s ): '''Get the handles openned by a specific process.''' parser = self.getParser( 'mem_handles', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to get the handles from, 0 for all processes' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.MEM_HANDLES_REQ, rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ), arguments )
def do_exec_oob_scan( self, s ): '''Scan one or more processes for out of bounds execution (thread out of known modules).''' parser = self.getParser( 'exec_oob_scan', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to scan, or "-1" for ALL processes' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.EXEC_OOB_REQ, rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ), arguments )
def do_file_info( self, s ): '''Retrieve information on a file from the host.''' parser = self.getParser( 'file_info', True ) parser.add_argument( 'file', type = unicode, help = 'file path to file to get info on' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.FILE_INFO_REQ, rSequence().addStringW( self.tags.base.FILE_PATH, arguments.file ), arguments )
def do_mem_find_handle( self, s ): '''Find the handles in any process that contain a specific substring.''' parser = self.getParser( 'mem_find_handle', True ) parser.add_argument( 'needle', type = unicode, help = 'substring of the handle names to get' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.MEM_FIND_HANDLE_REQ, rSequence().addStringW( self.tags.base.HANDLE_NAME, arguments.needle ), arguments )
def do_file_hash( self, s ): '''Hash a file from the host.''' parser = self.getParser( 'file_hash', True ) parser.add_argument( 'file', type = unicode, help = 'file path to hash' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.FILE_HASH_REQ, rSequence().addStringW( self.tags.base.FILE_PATH, arguments.file ), arguments )
def do_file_del(self, s): """Delete a file from the host.""" parser = self.getParser("file_del", True) parser.add_argument("file", type=unicode, help="file path to delete") arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking( self.tags.notification.FILE_DEL_REQ, rSequence().addStringW(self.tags.base.FILE_PATH, arguments.file), arguments, )
def do_hidden_module_scan( self, s ): '''Scan one or more processes for hidden modules.''' parser = self.getParser( 'hidden_module_scan', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to scan, or "-1" for ALL processes' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.HIDDEN_MODULE_REQ, rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ), arguments )
def do_mem_map(self, s): """Get the memory mapping of a specific process.""" parser = self.getParser("mem_map", True) parser.add_argument("pid", type=int, help="pid of the process to get the map from") arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking( self.tags.notification.MEM_MAP_REQ, rSequence().addInt32(self.tags.base.PROCESS_ID, arguments.pid), arguments, )
def do_remain_live( self, s ): '''Request the sensor remain in constant contact for the next X seconds.''' parser = self.getParser( 'remain_live', True ) parser.add_argument( 'seconds', type = int, help = 'number of seconds from now to remain live' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.REMAIN_LIVE_REQ, rSequence().addTimestamp( self.tags.base.EXPIRY, int( time.time() + arguments.seconds ) ), arguments )
def do_critical_del( self, s ): '''Tell the sensor to remove an event from the list of critical events.''' parser = self.getParser( 'critical_del', True ) parser.add_argument( 'event', type = eventArg, help = 'name of event to stop treating as critical' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.DEL_CRITICAL_EVENT_REQ, rSequence().addInt32( self.tags.hbs.NOTIFICATION_ID, arguments.event ), arguments )
def do_exfil_del( self, s ): '''Tell the sensor to stop exfiling specific event.''' parser = self.getParser( 'exfil_del', True ) parser.add_argument( 'event', type = eventArg, help = 'name of event to stop exfiling' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.DEL_EXFIL_EVENT_REQ, rSequence().addInt32( self.tags.hbs.NOTIFICATION_ID, arguments.event ), arguments )
def do_file_mov(self, s): """Move a file on the host.""" parser = self.getParser("file_mov", True) parser.add_argument("srcFile", type=unicode, help="source file path") parser.add_argument("dstFile", type=unicode, help="destination file path") arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking( self.tags.notification.FILE_MOV_REQ, rSequence() .addStringW(self.tags.base.FILE_PATH, arguments.srcFile) .addStringW(self.tags.base.FILE_NAME, arguments.dstFile), arguments, )
def do_file_mov( self, s ): '''Move a file on the host.''' parser = self.getParser( 'file_mov', True ) parser.add_argument( 'srcFile', type = unicode, help = 'source file path' ) parser.add_argument( 'dstFile', type = unicode, help = 'destination file path' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.FILE_MOV_REQ, rSequence().addStringW( self.tags.base.FILE_PATH, arguments.srcFile ) .addStringW( self.tags.base.FILE_NAME, arguments.dstFile ), arguments )
def drain(self): # Stop accepting new connections. if self.server is not None: self.server.close() # Ask all the clients to nicely disconnect. for aid, c in self.currentClients.items(): try: c.sendFrame(HcpModuleId.HCP, (rSequence().addInt8( Symbols.base.OPERATION, HcpOperations.DISCONNECT), )) except: pass # Wait for everyone to be out. while 0 != self.nConnected: self.log("still %d clients connected" % self.nConnected) self.sleep(5)
def do_mem_read(self, s): """Read the memory of a process at a specific address.""" parser = self.getParser("mem_read", True) parser.add_argument("pid", type=int, help="pid of the process to get the map from") parser.add_argument("baseAddr", type=hexArg, help="base address to read from, in HEX FORMAT") parser.add_argument("memSize", type=hexArg, help="number of bytes to read, in HEX FORMAT") arguments = self.parse(parser, s) if arguments is not None: self._executeHbsTasking( self.tags.notification.MEM_READ_REQ, rSequence() .addInt32(self.tags.base.PROCESS_ID, arguments.pid) .addInt64(self.tags.base.BASE_ADDRESS, arguments.baseAddr) .addInt32(self.tags.base.MEMORY_SIZE, arguments.memSize), arguments, )
def do_mem_read( self, s ): '''Read the memory of a process at a specific address.''' parser = self.getParser( 'mem_read', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to get the map from' ) parser.add_argument( 'baseAddr', type = hexArg, help = 'base address to read from, in HEX FORMAT' ) parser.add_argument( 'memSize', type = hexArg, help = 'number of bytes to read, in HEX FORMAT' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.notification.MEM_READ_REQ, rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ) .addInt64( self.tags.base.BASE_ADDRESS, arguments.baseAddr ) .addInt32( self.tags.base.MEMORY_SIZE, arguments.memSize ), arguments )
def do_exfil_add( self, s ): '''Tell the sensor to start exfiling specific event.''' parser = self.getParser( 'exfil_add', True ) parser.add_argument( 'event', type = eventArg, help = 'name of event to start exfiling' ) parser.add_argument( '-e', '--expire', type = int, required = False, dest = 'expire', help = 'number of seconds before stopping exfil of event' ) arguments = self.parse( parser, s ) if arguments is not None: data = ( rSequence().addInt32( self.tags.hbs.NOTIFICATION_ID, arguments.event ) .addTimestamp( self.tags.base.EXPIRY, int( time.time() + arguments.expire ) ) ) self._executeHbsTasking( self.tags.notification.ADD_EXFIL_EVENT_REQ, data, arguments )
def do_critical_add( self, s ): '''Tell the sensor to add an event to the list of critical events to beacon home.''' parser = self.getParser( 'critical_add', True ) parser.add_argument( 'event', type = eventArg, help = 'name of event to start treating as critical' ) parser.add_argument( '-e', '--expire', type = int, required = False, dest = 'expire', help = 'number of seconds before removing event from critical' ) arguments = self.parse( parser, s ) if arguments is not None: data = ( rSequence().addInt32( self.tags.hbs.NOTIFICATION_ID, arguments.event ) .addTimestamp( self.tags.base.EXPIRY, int( time.time() + arguments.expire ) ) ) self._executeHbsTasking( self.tags.notification.ADD_CRITICAL_EVENT_REQ, data, arguments )
def do_mem_find_string(self, s): """Find the specific strings in a specific process.""" parser = self.getParser("mem_find_string", True) parser.add_argument("pid", type=int, help="pid of the process to search in") parser.add_argument( "-s", "--strings", type=unicode, required=True, nargs="*", dest="strings", help="list of strings to look for", ) arguments = self.parse(parser, s) if arguments is not None: seq = rSequence().addInt32(self.tags.base.PROCESS_ID, arguments.pid) l = rList() for s in arguments.strings: l.addStringW(self.tags.base.STRING, s) seq.addList(self.tags.base.STRINGSW, l) self._executeHbsTasking(self.tags.notification.MEM_FIND_STRING_REQ, seq, arguments)
def do_dir_list( self, s ): '''Get the directory listing.''' parser = self.getParser( 'dir_list', True ) parser.add_argument( 'rootDir', type = unicode, help = 'the root directory where to begin the listing from' ) parser.add_argument( 'fileExp', type = unicode, help = 'a file name expression supporting basic wildcards like * and ?' ) parser.add_argument( '-d', '--depth', dest = 'depth', required = False, default = 0, help = 'optional maximum depth of the listing, defaults to a single level' ) arguments = self.parse( parser, s ) if arguments is not None: self._executeHbsTasking( self.tags.hbs.NOTIFICATION_DIR_LIST_REQ, rSequence().addStringW( self.tags.base.FILE_PATH, arguments.fileExp ) .addStringW( self.tags.base.DIRECTORY_PATH, arguments.rootDir ) .addInt32( self.tags.base.DIRECTORY_LIST_DEPTH, arguments.depth ), arguments )
def do_mem_find_string( self, s ): '''Find the specific strings in a specific process.''' parser = self.getParser( 'mem_find_string', True ) parser.add_argument( 'pid', type = int, help = 'pid of the process to search in' ) parser.add_argument( '-s', '--strings', type = unicode, required = True, nargs = '*', dest = 'strings', help = 'list of strings to look for' ) arguments = self.parse( parser, s ) if arguments is not None: seq = rSequence().addInt32( self.tags.base.PROCESS_ID, arguments.pid ) l = rList() for s in arguments.strings: l.addStringW( self.tags.base.STRING, s ) seq.addList( self.tags.base.STRINGSW, l ) self._executeHbsTasking( self.tags.notification.MEM_FIND_STRING_REQ, seq, arguments )
def timeSyncMessage(self): return (rSequence().addInt8( Symbols.base.OPERATION, HcpOperations.SET_GLOBAL_TIME).addTimestamp( Symbols.base.TIMESTAMP, int(time.time())))
def handleNewClient(self, socket, address): if not self.isOpen: return self.nConnected += 1 aid = None tmpBytesReceived = 0 bufferedOutput = None self.log('New connection from %s:%s' % address) try: c = _ClientContext(self, socket) moduleId, headers, _ = c.recvFrame(timeout=30.0) if HcpModuleId.HCP != moduleId: raise DisconnectException('Headers not from expected module') if headers is None: raise DisconnectException('Error deserializing headers') headers = headers[0] self.log('Headers decoded, validating connection') hostName = headers.get('base.HOST_NAME', None) internalIp = headers.get('base.IP_ADDRESS', None) hcpHash = headers.get('base.HASH', None) hcpCrashContext = headers.get('hcp.CRASH_CONTEXT', None) if hcpCrashContext is not None: self.zInc('cc_received') # Use the address in the client context since it was received from the # proxy headers and therefore is the correct original source. externalIp = c.address[0] c.hostName = hostName c.int_ip = internalIp c.ext_ip = externalIp aid = AgentId(headers['base.HCP_IDENT']) if aid.org_id is None or aid.ins_id is None or aid.platform is None or aid.architecture is None: aidInfo = str(aid) if 0 == len(aidInfo): aidInfo = str(headers) raise DisconnectException('Invalid sensor id: %s' % aidInfo) if aid.sensor_id is None: self.log('Sensor requires enrollment') resp = self.enrollmentManager.request('enroll', { 'aid': aid.asString(), 'public_ip': externalIp, 'internal_ip': internalIp, 'host_name': hostName }, timeout=30) if not resp.isSuccess or 'aid' not in resp.data or resp.data[ 'aid'] is None: raise DisconnectException( 'Sensor could not be enrolled, come back later') aid = AgentId(resp.data['aid']) enrollmentToken = resp.data['token'] confBuffer = resp.data['conf'] confBufferSig = resp.data['conf_sig'] self.log('Sending sensor enrollment to %s' % aid.asString()) c.sendFrame(HcpModuleId.HCP, (rSequence().addInt8( Symbols.base.OPERATION, HcpOperations.SET_HCP_CONF).addBuffer( Symbols.hcp.CONFIGURATION, confBuffer).addBuffer( Symbols.base.SIGNATURE, confBufferSig), rSequence().addInt8( Symbols.base.OPERATION, HcpOperations.SET_HCP_ID).addSequence( Symbols.base.HCP_IDENT, aid.toJson()).addBuffer( Symbols.hcp.ENROLLMENT_TOKEN, enrollmentToken))) confBuffer = None confBufferSig = None else: enrollmentToken = headers.get('hcp.ENROLLMENT_TOKEN', None) resp = self.enrollmentManager.request('authorize', { 'aid': aid.asString(), 'token': enrollmentToken, 'hash': hcpHash }, timeout=10) if not resp.isSuccess or not resp.data.get( 'is_authorized', False): raise DisconnectException('Could not authorize %s' % aid) self.log('Valid client connection') # Eventually sync the clocks at recurring intervals c.sendFrame(HcpModuleId.HCP, (self.timeSyncMessage(), )) c.setAid(aid) self.currentClients[aid.sensor_id] = c self.zSet('clients', len(self.currentClients)) newStateMsg = { 'aid': aid.asString(), 'endpoint': self.name, 'ext_ip': externalIp, 'int_ip': internalIp, 'hostname': hostName, 'connection_id': c.connId } self.stateChanges.shoot('live', newStateMsg, timeout=30) self.sensorDir.broadcast('live', newStateMsg) del (newStateMsg) resp = self.tagging.request('get_tags', {'sid': aid.sensor_id}, timeout=2) if resp.isSuccess: c.tags = resp.data.get('tags', {}).values()[0].keys() self.log('Retrieved tags %s for %s' % (c.tags, aid.asString())) self.log('Client %s registered, beginning to receive data' % aid.asString()) lastTransferReport = time.time() frameIndex = 0 bufferedOutput = LimitedQPSBuffer( self.sensorMaxQps, cbLog=lambda x: self.log("%s %s" % (aid.asString(), x))) while True: moduleId, messages, nRawBytes = c.recvFrame(timeout=60 * 11) tmpBytesReceived += nRawBytes if 10 == frameIndex: now = time.time() if now > lastTransferReport + (60 * 10): self.sensorDir.broadcast( 'transfered', { 'aid': aid.asString(), 'bytes_transfered': tmpBytesReceived }) self.stateChanges.shoot( 'transfered', { 'aid': aid.asString(), 'bytes_transfered': tmpBytesReceived }) tmpBytesReceived = 0 lastTransferReport = now frameIndex = 0 else: frameIndex += 1 handler = self.moduleHandlers.get(moduleId, None) if handler is None: self.log('Received data for unknown module') else: bufferedOutput.add(handler, c, messages) except Exception as e: if type(e) is not DisconnectException: self.log('Exception while processing: %s' % str(e)) self.log(traceback.format_exc()) raise else: self.log('Disconnecting: %s' % str(e)) finally: if aid is not None: if aid.sensor_id in self.currentClients: del (self.currentClients[aid.sensor_id]) self.sensorDir.broadcast( 'transfered', { 'aid': aid.asString(), 'bytes_transfered': tmpBytesReceived }) self.stateChanges.shoot( 'transfered', { 'aid': aid.asString(), 'bytes_transfered': tmpBytesReceived }) newStateMsg = { 'aid': aid.asString(), 'endpoint': self.name, 'connection_id': c.connId } self.stateChanges.shoot('dead', newStateMsg, timeout=30) self.sensorDir.broadcast('dead', newStateMsg) del (newStateMsg) self.log('Connection terminated: %s' % aid.asString()) self.zSet('clients', len(self.currentClients)) else: self.log('Connection terminated: %s:%s' % address) if bufferedOutput is not None: qSize = bufferedOutput.size() if 0 != qSize: self.log('Waiting for queue of size %s to flush for %s' % (qSize, aid.asString())) bufferedOutput.close() if 0 != qSize: self.log('Queue for %s finished flushing' % aid.asString()) self.nConnected -= 1