def initialize(self): """ Method gets called when socket-io is initialized. """ self.univ = universal.Universal( ) #save the reference to Universal for optimized access
def prepare(self): """ Public method to prepare the job for execution. This sets the execution timestamp. Returns: Execution start timestamp of the job """ t = int(time.time() * 1000) self.exec_details['timestamp'] = t #set the exec timestamp if self.is_whitelisted == True: #if the job is whitelisted _log.debug('Executing activity(%s @ %d)' % (self.exec_details['_id'], t)) #if it is not a plugin job, then we create a temperory file to capture the output #a plugin job return the data directly if not self.is_plugin: self.exec_details['output'] = tempfile.NamedTemporaryFile(dir = universal.Universal().temp_path) self.status = JobStatus.RUNNING #change the state to running else: _log.info('Activity %s is blocked by whitelist' % self.exec_details['_id']) self.status = JobStatus.BLOCKED #change the state to blocked return t
def __init__(self, activity): """ Constructor Args: activity: dict representing the activity to be executed """ self.is_whitelisted = activity[ 'is_whitelisted'] #is this job allowed to execute self.exec_timestamp = activity[ 'next_exec_timestamp'] #timestamp at which the job should execute self.status = JobStatus.INITIALIZED #current job state #is this job a plugin or a commandline; this can also point to a generator instance which indicates a job that has been started already #so, False means its a commandline job, anything else means it is a plugin job self.plugin = True if activity['details'].get( 'service') == 'Plugins' else False #dict containing job execution details self.exec_details = { 'timestamp': 0, #actual timestamp when the job started 'output': None, #output of the job, file handle for commandline job, dict for successful pugin execution, str for failed pugin execution. 'pid': -1, #process id if it is a commandline job 'return_code': 0, #return code of the job 'activity': activity['details'], #activity the job represents } self.univ = universal.Universal( ) #reference to Universal for optimized access
def __init__(self): """ Constructor """ self.univ = universal.Universal( ) #reference to Universal for optimized access self.env_variables = { } #env variables received from the server to execute commands ThreadEx.__init__(self) #inititalize thread base class #inititalize process base class; refer execute.sh to read more about arguments to be passed in exec_args = [ 'bash', '%s/src/execute.sh' % self.univ.exe_path, os.path.realpath(sys.modules['__main__'].__file__) ] os.isatty(sys.stdin.fileno()) or exec_args.append('1') WorkerProcess.__init__(self, *exec_args) self.line_ending = '\r' self.daemon = True #run this thread as daemon as it should not block agent from shutting down self.univ.event_dispatcher.bind( 'terminate', self.stop ) #bind to terminate event so that we can terminate bash process #use the job timeout defined in the config if we have one try: self.timeout = self.univ.config.sealion.commandTimeout except: self.timeout = 30 self.timeout = int(self.timeout * 1000) #convert to millisec
def run(self): """ Method runs in the daemon. """ self.set_procname(self.daemon_name + ('d' if self.daemon_name[-1] != 'd' else '')) #set process name for display purpose crash_dump_details = self.get_crash_dump_details( ) #get crash dump details helper.terminatehook = self.termination_hook #set the termination hook called whenever agent shutdown disgracefully if crash_dump_details[1] > 0: #start thread to send crash dump _log.info('Found %d dumps' % crash_dump_details[1]) ThreadEx(target=self.send_crash_dumps, name='CrashDumpSender').start() if crash_dump_details[ 0] == True: #crash loop detected. start agent in update only mode _log.info( 'Crash loop detected; Starting agent in update-only mode') universal.Universal().is_update_only_mode = True import main main.stop_stream_logging() #stop logging on stdout/stderr main.run() #start executing agent
def main(directory, command_interval): signal.signal( signal.SIGQUIT, sigquit_handler) #install SIGQUIT handler so that the program can stop sys.stderr = sys.stdout #as we are using log module and we want the output to be in stdout, redirect logging.basicConfig(level=logging.DEBUG, format='%(message)s') service.set_user() #set the user and group for the process univ, seperator = universal.Universal(), '\n' #export the environment variables os.environ.update( univ.config.sealion.get_dict((['config', 'envVariables'], {}))['envVariables']) os.environ.update(univ.config.sealion.get_dict(('env', {}))['env']) os.environ.update({'COMMAND_INTERVAL': unicode(command_interval)}) try: os.chdir(os.path.realpath(directory)) log.debug( 'SIGQUIT(Ctrl-\\) to exit; SIGINT(Ctrl-C) to abort current operation' ) log.debug('Working directory: %s' % os.path.realpath(directory)) #loop through the content of the directory for activity in os.listdir('./'): try: #consider only *.sh files if activity[-3:] != '.sh' or not os.path.isfile(activity): continue seperator and log.debug(seperator) output, status = execute(activity) #execute and get the output if not output: continue metrics = {} #loop through the contents of the metric folder for the activity for metric in os.listdir(activity[:-3]): try: #consider only *.py files if metric[-3:] != '.py': continue #read the parser code from the file with open(activity[:-3] + '/' + metric) as f: metrics[metric] = {'parser': f.read()} except: pass extract.extract_metrics(output, status, metrics, activity) #extract metrics seperator = '%s\n' % ('_' * 50) except: pass except Exception as e: log.error('Error: %s', unicode(e))
def __init__(self): """ Constructor. """ self.univ = universal.Universal() #save a reference to Universal for optimized access self.off_store = OfflineStore() #offline store self.realtime_sender = RealtimeSender(self.off_store) #real time sender self.historic_sender = HistoricSender(self.off_store) #historic sender
def __init__(self): """ Constructor """ ThreadEx.__init__(self) #initialize base class self.univ = universal.Universal( ) #save the reference to Universal for optimized access self.daemon = True #set the daemon flag as we dont want this thread to block agent shutdown
def __init__(self): """ Constructor """ ThreadEx.__init__(self) #initialize the base class self.job_producer = JobProducer() #job producer self.univ = universal.Universal() #save the reference to Universal for optimized access self.name = '%s-%d' % (self.__class__.__name__, JobConsumer.unique_id) #set the name JobConsumer.unique_id += 1 #increment the id
def __init__(self): """ Constructor """ ThreadEx.__init__(self) #initialize the base class self.univ = universal.Universal() #save the reference to Universal for optimized access self.is_stop = False #flag determines to stop the execution of controller self.main_thread = threading.current_thread() #reference for main thread self.updater = None #updater thread self.updater_lock = threading.RLock() #thread lock for updating agent
def __init__(self, *args, **kwargs): """ Constructor """ requests.Session.__init__(self, *args, **kwargs) #initialize the base class self.univ = universal.Universal() #save the reference to Universal for optimized access self.stop_status = Status.SUCCESS #reason for stopping self.authenticate_status = AuthStatus.UNAUTHORIZED #authentication status self.auth_lock = threading.RLock() #lock for authentication status self.is_conn_err = False #last api call returned error self.session_conflict_count = 0 #counter to keep track of subsequant session conflicts
def __init__(self): """ Constructor. """ ThreadEx.__init__(self) #initialize base class self.sio = None #socket-io instance self.univ = universal.Universal( ) #save the reference to Universal for optimized access self.is_stop = False #flag tells whether to stop the thread. self.daemon = True #run this thread as daemon as it should not block agent from shutting down self.is_disconnected = False #whether socket-io is disconnected self.session_id = '' #session id to verify handshake error self.update_heartbeat() #set the heardbeat
def save_dump(self, stack_trace): """ Method to save the stack trace as a crash dump. Args: stack_trace: stack trace of exception. Returns: Path to the crash dump on success else False """ univ = universal.Universal() #get Universal timestamp = int(time.time() * 1000) #timestamp for the unique crash dump filename path = self.crash_dump_path + ( 'sealion-%s-%d.dmp' % (univ.config.agent.agentVersion, timestamp)) #crash dump filename f = None try: helper.Utils.get_safe_path( path) #create dump directory if it is not available #dict continaing crash dump details report = { 'timestamp': timestamp, 'stack': stack_trace, 'orgToken': univ.config.agent.orgToken, '_id': univ.config.agent.get(['config', '_id']), 'os': { 'pythonVersion': univ.details['pythonVersion'] }, 'process': { 'uid': os.getuid(), 'gid': os.getgid(), 'uptime': int(univ.get_run_time() * 1000), 'agentVersion': univ.config.agent.agentVersion, 'isProxy': univ.details['isProxy'] } } #write dump f = open(path, 'w') json.dump(report, f) except: return None finally: f and f.close() return path
def __init__(self, off_store): """ Constructor. Args: off_store: offline store instance """ ThreadEx.__init__(self) #initialize base class self.univ = universal.Universal() #save a reference to Universal for optimized access self.off_store = off_store #offline store instance to be used self.queue_max_size = 150 #max sending queue count self.ping_interval = 10 #the ping interval for retry after an failed api request self.queue = queue.Queue(self.queue_max_size) #sending queue self.last_ping_time = int(time.time()) #saves the last time api was pinged
def __init__(self): """ Constructor. """ ThreadEx.__init__(self) #initialize base class self.univ = universal.Universal() #save a reference to Universal for optimized access self.db_file = self.univ.db_path #sqlite db file path; same property is used as absolute path to the filename once offline store is started self.conn = None #sqlite db connection; self.conn_event = threading.Event() #event to synchronize connection made in the thread self.task_queue = queue.Queue() #the task queue used to feed the operations to thread self.is_bulk_insert = False #whether to use transaction around contigous insert statements self.pending_insert_row_count = 0 #number of rows pending to be inserted in a transaction self.select_max_timestamp = int(time.time() * 1000) #timestamp limit for retreival of rows self.select_timestamp_lock = threading.RLock() #thread lock for updating timestamp limit for retreival of rows
def dump_stack_traces(): """ Function to dump the stack trace of all the threads to a file """ trace = helper.Utils.get_stack_trace() #get the stack trace of all the threads f, timestamp = None, int(time.time() * 1000) try: path = helper.Utils.get_safe_path(universal.Universal().exe_path + ('var/log/stack-trace-%d.log' % timestamp)) #unique filename for stack trace f = open(path, 'w') f.write(trace) _log.info('Stack trace saved at %s' % path) except Exception as e: _log.error('Failed to save stack trace; %s' % unicode(e)) finally: f and f.close()
def __init__(self, store): """ Constructor Args: store: Storage instance. """ ThreadEx.__init__(self) #initialize the base class self.univ = universal.Universal() #store reference to Universal for optmized access self.activities_lock = threading.RLock() #threading lock for updating activities self.activities = {} #dict of activities self.queue = queue.Queue() #job queue self.sleep_interval = 5 #how much time should the thread sleep before scheduling self.store = store #storage instance self.consumer_count = 0 #total number of job consumers running self.executer = Executer() #executer instance for running commandline activities self.univ.event_dispatcher.bind('get_activity_funct', self.get_activity_funct)
def __init__(self): """ Constructor """ self.univ = universal.Universal() self.extract_lock = threading.RLock( ) #for limiting access to extractor process #use the metric timeout defined in the config if we have one try: self.timeout = self.univ.config.sealion.metricTimeout except: self.timeout = 2 #initialize the worker process with the python executable and arguments, this doesn't start the process WorkerProcess.__init__(self, sys.executable, '%s/src/extract.py' % self.univ.exe_path, '%s' % self.timeout)
def __init__(self): """ Constructor """ ThreadEx.__init__(self) #inititalize the base class self.exec_process = None #bash process instance self.process_lock = threading.RLock() #thread lock for bash process instance self.exec_count = 0 #2254 total number of commands executed in the bash process self.is_stop = False #stop flag for the thread self.univ = universal.Universal() #reference to Universal for optimized access self.daemon = True #run this thread as daemon as it should not block agent from shutting down self.univ.event_dispatcher.bind('terminate', self.stop) #bind to terminate event so that we can terminate bash process #use the job timeout defined in the config if we have one try: self.timeout = self.univ.config.sealion.commandTimeout except: self.timeout = 30 self.timeout = int(self.timeout * 1000) #convert to millisec
def get_crash_dump_details(self): """ Method to get crash dump count. It also reports any crash loop by examining the file timestamp. Returns: Tupple (is crah loop, crash dump count) """ univ = universal.Universal() #get Universal t = int(time.time()) #current epoch time for crash loop detection crash_loop_timeout = self.crash_loop_count * self.crash_loop_timeout #time span for crash loop detection file_count, loop_file_count = 0, 0 #crash loop is detected only for the current agent version running loop_regex = self.crash_dump_pattern % univ.config.agent.agentVersion.replace( '.', r'\.') try: for f in os.listdir( self.crash_dump_path ): #loop though files in the crash dump directory #if it is a valid crash dump file name if os.path.isfile(self.crash_dump_path + f) and re.match( self.crash_dump_pattern % self.agent_version_regex, f) != None: file_count += 1 #is this file contribute to crash loop if re.match( loop_regex, f) != None and t - os.path.getmtime( self.crash_dump_path + f) < crash_loop_timeout: loop_file_count += 1 except: pass return (loop_file_count >= 5, file_count)
def send_crash_dumps(self): """ Method to send all crash dumps to server. This method runs in a seperate thread. """ import api univ = universal.Universal() #get Universal #how much time the crash dump sender wait before start sending. #this is required not to affect crash loop detection, since crash loop detection is done by checking number crash dumps generated in a span of time crash_dump_timeout = (self.crash_loop_count * self.crash_loop_timeout) + 10 _log.debug('CrashDumpSender waiting for stop event for %d seconds' % crash_dump_timeout) univ.stop_event.wait(crash_dump_timeout) try: for file in os.listdir( self.crash_dump_path ): #loop though files in the crash dump directory file_name = self.crash_dump_path + file #is this a valid crash dump filename if os.path.isfile(file_name) and re.match( self.crash_dump_pattern % self.agent_version_regex, file) != None: report = None while 1: if univ.stop_event.is_set(): #do we need to stop now _log.debug('CrashDumpSender received stop event') return #read the report from the dump, or retry the report report = report if report != None else self.read_dump( file_name) if report == None or api.is_not_connected( api.unauth_session.send_crash_report( report)) == False: #send the dump break _log.debug( 'CrashDumpSender waiting for stop event for 10 seconds' ) univ.stop_event.wait( 10) #on failure, wait for some time try: os.remove(file_name) #remove the dump as we sent it _log.info('Removed dump %s' % file_name) except Exception as e: _log.error('Failed to remove dump %s; %s' % (file_name, unicode(e))) if univ.stop_event.is_set(): #do we need to stop now _log.debug('CrashDumpSender received stop event') return except: pass