def initialize(self):
"""
Method gets called when socket-io is initialized.
"""
self.univ = universal.Universal(
) #save the reference to Universal for optimized access
def prepare(self):
"""
Public method to prepare the job for execution.
This sets the execution timestamp.
Returns:
Execution start timestamp of the job
"""
t = int(time.time() * 1000)
self.exec_details['timestamp'] = t #set the exec timestamp
if self.is_whitelisted == True: #if the job is whitelisted
_log.debug('Executing activity(%s @ %d)' % (self.exec_details['_id'], t))
#if it is not a plugin job, then we create a temperory file to capture the output
#a plugin job return the data directly
if not self.is_plugin:
self.exec_details['output'] = tempfile.NamedTemporaryFile(dir = universal.Universal().temp_path)
self.status = JobStatus.RUNNING #change the state to running
else:
_log.info('Activity %s is blocked by whitelist' % self.exec_details['_id'])
self.status = JobStatus.BLOCKED #change the state to blocked
return t
def __init__(self, activity):
"""
Constructor
Args:
activity: dict representing the activity to be executed
"""
self.is_whitelisted = activity[
'is_whitelisted'] #is this job allowed to execute
self.exec_timestamp = activity[
'next_exec_timestamp'] #timestamp at which the job should execute
self.status = JobStatus.INITIALIZED #current job state
#is this job a plugin or a commandline; this can also point to a generator instance which indicates a job that has been started already
#so, False means its a commandline job, anything else means it is a plugin job
self.plugin = True if activity['details'].get(
'service') == 'Plugins' else False
#dict containing job execution details
self.exec_details = {
'timestamp': 0, #actual timestamp when the job started
'output':
None, #output of the job, file handle for commandline job, dict for successful pugin execution, str for failed pugin execution.
'pid': -1, #process id if it is a commandline job
'return_code': 0, #return code of the job
'activity': activity['details'], #activity the job represents
}
self.univ = universal.Universal(
) #reference to Universal for optimized access
def __init__(self):
"""
Constructor
"""
self.univ = universal.Universal(
) #reference to Universal for optimized access
self.env_variables = {
} #env variables received from the server to execute commands
ThreadEx.__init__(self) #inititalize thread base class
#inititalize process base class; refer execute.sh to read more about arguments to be passed in
exec_args = [
'bash',
'%s/src/execute.sh' % self.univ.exe_path,
os.path.realpath(sys.modules['__main__'].__file__)
]
os.isatty(sys.stdin.fileno()) or exec_args.append('1')
WorkerProcess.__init__(self, *exec_args)
self.line_ending = '\r'
self.daemon = True #run this thread as daemon as it should not block agent from shutting down
self.univ.event_dispatcher.bind(
'terminate', self.stop
) #bind to terminate event so that we can terminate bash process
#use the job timeout defined in the config if we have one
try:
self.timeout = self.univ.config.sealion.commandTimeout
except:
self.timeout = 30
self.timeout = int(self.timeout * 1000) #convert to millisec
def run(self):
"""
Method runs in the daemon.
"""
self.set_procname(self.daemon_name +
('d' if self.daemon_name[-1] != 'd' else
'')) #set process name for display purpose
crash_dump_details = self.get_crash_dump_details(
) #get crash dump details
helper.terminatehook = self.termination_hook #set the termination hook called whenever agent shutdown disgracefully
if crash_dump_details[1] > 0: #start thread to send crash dump
_log.info('Found %d dumps' % crash_dump_details[1])
ThreadEx(target=self.send_crash_dumps,
name='CrashDumpSender').start()
if crash_dump_details[
0] == True: #crash loop detected. start agent in update only mode
_log.info(
'Crash loop detected; Starting agent in update-only mode')
universal.Universal().is_update_only_mode = True
import main
main.stop_stream_logging() #stop logging on stdout/stderr
main.run() #start executing agent
def main(directory, command_interval):
signal.signal(
signal.SIGQUIT,
sigquit_handler) #install SIGQUIT handler so that the program can stop
sys.stderr = sys.stdout #as we are using log module and we want the output to be in stdout, redirect
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
service.set_user() #set the user and group for the process
univ, seperator = universal.Universal(), '\n'
#export the environment variables
os.environ.update(
univ.config.sealion.get_dict((['config',
'envVariables'], {}))['envVariables'])
os.environ.update(univ.config.sealion.get_dict(('env', {}))['env'])
os.environ.update({'COMMAND_INTERVAL': unicode(command_interval)})
try:
os.chdir(os.path.realpath(directory))
log.debug(
'SIGQUIT(Ctrl-\\) to exit; SIGINT(Ctrl-C) to abort current operation'
)
log.debug('Working directory: %s' % os.path.realpath(directory))
#loop through the content of the directory
for activity in os.listdir('./'):
try:
#consider only *.sh files
if activity[-3:] != '.sh' or not os.path.isfile(activity):
continue
seperator and log.debug(seperator)
output, status = execute(activity) #execute and get the output
if not output:
continue
metrics = {}
#loop through the contents of the metric folder for the activity
for metric in os.listdir(activity[:-3]):
try:
#consider only *.py files
if metric[-3:] != '.py':
continue
#read the parser code from the file
with open(activity[:-3] + '/' + metric) as f:
metrics[metric] = {'parser': f.read()}
except:
pass
extract.extract_metrics(output, status, metrics,
activity) #extract metrics
seperator = '%s\n' % ('_' * 50)
except:
pass
except Exception as e:
log.error('Error: %s', unicode(e))
def __init__(self):
"""
Constructor.
"""
self.univ = universal.Universal() #save a reference to Universal for optimized access
self.off_store = OfflineStore() #offline store
self.realtime_sender = RealtimeSender(self.off_store) #real time sender
self.historic_sender = HistoricSender(self.off_store) #historic sender
def __init__(self):
"""
Constructor
"""
ThreadEx.__init__(self) #initialize base class
self.univ = universal.Universal(
) #save the reference to Universal for optimized access
self.daemon = True #set the daemon flag as we dont want this thread to block agent shutdown
def __init__(self):
"""
Constructor
"""
ThreadEx.__init__(self) #initialize the base class
self.job_producer = JobProducer() #job producer
self.univ = universal.Universal() #save the reference to Universal for optimized access
self.name = '%s-%d' % (self.__class__.__name__, JobConsumer.unique_id) #set the name
JobConsumer.unique_id += 1 #increment the id
def __init__(self):
"""
Constructor
"""
ThreadEx.__init__(self) #initialize the base class
self.univ = universal.Universal() #save the reference to Universal for optimized access
self.is_stop = False #flag determines to stop the execution of controller
self.main_thread = threading.current_thread() #reference for main thread
self.updater = None #updater thread
self.updater_lock = threading.RLock() #thread lock for updating agent
def __init__(self, *args, **kwargs):
"""
Constructor
"""
requests.Session.__init__(self, *args, **kwargs) #initialize the base class
self.univ = universal.Universal() #save the reference to Universal for optimized access
self.stop_status = Status.SUCCESS #reason for stopping
self.authenticate_status = AuthStatus.UNAUTHORIZED #authentication status
self.auth_lock = threading.RLock() #lock for authentication status
self.is_conn_err = False #last api call returned error
self.session_conflict_count = 0 #counter to keep track of subsequant session conflicts
def __init__(self):
"""
Constructor.
"""
ThreadEx.__init__(self) #initialize base class
self.sio = None #socket-io instance
self.univ = universal.Universal(
) #save the reference to Universal for optimized access
self.is_stop = False #flag tells whether to stop the thread.
self.daemon = True #run this thread as daemon as it should not block agent from shutting down
self.is_disconnected = False #whether socket-io is disconnected
self.session_id = '' #session id to verify handshake error
self.update_heartbeat() #set the heardbeat
def save_dump(self, stack_trace):
"""
Method to save the stack trace as a crash dump.
Args:
stack_trace: stack trace of exception.
Returns:
Path to the crash dump on success else False
"""
univ = universal.Universal() #get Universal
timestamp = int(time.time() *
1000) #timestamp for the unique crash dump filename
path = self.crash_dump_path + (
'sealion-%s-%d.dmp' %
(univ.config.agent.agentVersion, timestamp)) #crash dump filename
f = None
try:
helper.Utils.get_safe_path(
path) #create dump directory if it is not available
#dict continaing crash dump details
report = {
'timestamp': timestamp,
'stack': stack_trace,
'orgToken': univ.config.agent.orgToken,
'_id': univ.config.agent.get(['config', '_id']),
'os': {
'pythonVersion': univ.details['pythonVersion']
},
'process': {
'uid': os.getuid(),
'gid': os.getgid(),
'uptime': int(univ.get_run_time() * 1000),
'agentVersion': univ.config.agent.agentVersion,
'isProxy': univ.details['isProxy']
}
}
#write dump
f = open(path, 'w')
json.dump(report, f)
except:
return None
finally:
f and f.close()
return path
def __init__(self, off_store):
"""
Constructor.
Args:
off_store: offline store instance
"""
ThreadEx.__init__(self) #initialize base class
self.univ = universal.Universal() #save a reference to Universal for optimized access
self.off_store = off_store #offline store instance to be used
self.queue_max_size = 150 #max sending queue count
self.ping_interval = 10 #the ping interval for retry after an failed api request
self.queue = queue.Queue(self.queue_max_size) #sending queue
self.last_ping_time = int(time.time()) #saves the last time api was pinged
def __init__(self):
"""
Constructor.
"""
ThreadEx.__init__(self) #initialize base class
self.univ = universal.Universal() #save a reference to Universal for optimized access
self.db_file = self.univ.db_path #sqlite db file path; same property is used as absolute path to the filename once offline store is started
self.conn = None #sqlite db connection;
self.conn_event = threading.Event() #event to synchronize connection made in the thread
self.task_queue = queue.Queue() #the task queue used to feed the operations to thread
self.is_bulk_insert = False #whether to use transaction around contigous insert statements
self.pending_insert_row_count = 0 #number of rows pending to be inserted in a transaction
self.select_max_timestamp = int(time.time() * 1000) #timestamp limit for retreival of rows
self.select_timestamp_lock = threading.RLock() #thread lock for updating timestamp limit for retreival of rows
def dump_stack_traces():
"""
Function to dump the stack trace of all the threads to a file
"""
trace = helper.Utils.get_stack_trace() #get the stack trace of all the threads
f, timestamp = None, int(time.time() * 1000)
try:
path = helper.Utils.get_safe_path(universal.Universal().exe_path + ('var/log/stack-trace-%d.log' % timestamp)) #unique filename for stack trace
f = open(path, 'w')
f.write(trace)
_log.info('Stack trace saved at %s' % path)
except Exception as e:
_log.error('Failed to save stack trace; %s' % unicode(e))
finally:
f and f.close()
def __init__(self, store):
"""
Constructor
Args:
store: Storage instance.
"""
ThreadEx.__init__(self) #initialize the base class
self.univ = universal.Universal() #store reference to Universal for optmized access
self.activities_lock = threading.RLock() #threading lock for updating activities
self.activities = {} #dict of activities
self.queue = queue.Queue() #job queue
self.sleep_interval = 5 #how much time should the thread sleep before scheduling
self.store = store #storage instance
self.consumer_count = 0 #total number of job consumers running
self.executer = Executer() #executer instance for running commandline activities
self.univ.event_dispatcher.bind('get_activity_funct', self.get_activity_funct)
def __init__(self):
"""
Constructor
"""
self.univ = universal.Universal()
self.extract_lock = threading.RLock(
) #for limiting access to extractor process
#use the metric timeout defined in the config if we have one
try:
self.timeout = self.univ.config.sealion.metricTimeout
except:
self.timeout = 2
#initialize the worker process with the python executable and arguments, this doesn't start the process
WorkerProcess.__init__(self, sys.executable,
'%s/src/extract.py' % self.univ.exe_path,
'%s' % self.timeout)
def __init__(self):
"""
Constructor
"""
ThreadEx.__init__(self) #inititalize the base class
self.exec_process = None #bash process instance
self.process_lock = threading.RLock() #thread lock for bash process instance
self.exec_count = 0 #2254 total number of commands executed in the bash process
self.is_stop = False #stop flag for the thread
self.univ = universal.Universal() #reference to Universal for optimized access
self.daemon = True #run this thread as daemon as it should not block agent from shutting down
self.univ.event_dispatcher.bind('terminate', self.stop) #bind to terminate event so that we can terminate bash process
#use the job timeout defined in the config if we have one
try:
self.timeout = self.univ.config.sealion.commandTimeout
except:
self.timeout = 30
self.timeout = int(self.timeout * 1000) #convert to millisec
def get_crash_dump_details(self):
"""
Method to get crash dump count. It also reports any crash loop by examining the file timestamp.
Returns:
Tupple (is crah loop, crash dump count)
"""
univ = universal.Universal() #get Universal
t = int(time.time()) #current epoch time for crash loop detection
crash_loop_timeout = self.crash_loop_count * self.crash_loop_timeout #time span for crash loop detection
file_count, loop_file_count = 0, 0
#crash loop is detected only for the current agent version running
loop_regex = self.crash_dump_pattern % univ.config.agent.agentVersion.replace(
'.', r'\.')
try:
for f in os.listdir(
self.crash_dump_path
): #loop though files in the crash dump directory
#if it is a valid crash dump file name
if os.path.isfile(self.crash_dump_path + f) and re.match(
self.crash_dump_pattern % self.agent_version_regex,
f) != None:
file_count += 1
#is this file contribute to crash loop
if re.match(
loop_regex, f) != None and t - os.path.getmtime(
self.crash_dump_path + f) < crash_loop_timeout:
loop_file_count += 1
except:
pass
return (loop_file_count >= 5, file_count)
def send_crash_dumps(self):
"""
Method to send all crash dumps to server.
This method runs in a seperate thread.
"""
import api
univ = universal.Universal() #get Universal
#how much time the crash dump sender wait before start sending.
#this is required not to affect crash loop detection, since crash loop detection is done by checking number crash dumps generated in a span of time
crash_dump_timeout = (self.crash_loop_count *
self.crash_loop_timeout) + 10
_log.debug('CrashDumpSender waiting for stop event for %d seconds' %
crash_dump_timeout)
univ.stop_event.wait(crash_dump_timeout)
try:
for file in os.listdir(
self.crash_dump_path
): #loop though files in the crash dump directory
file_name = self.crash_dump_path + file
#is this a valid crash dump filename
if os.path.isfile(file_name) and re.match(
self.crash_dump_pattern % self.agent_version_regex,
file) != None:
report = None
while 1:
if univ.stop_event.is_set(): #do we need to stop now
_log.debug('CrashDumpSender received stop event')
return
#read the report from the dump, or retry the report
report = report if report != None else self.read_dump(
file_name)
if report == None or api.is_not_connected(
api.unauth_session.send_crash_report(
report)) == False: #send the dump
break
_log.debug(
'CrashDumpSender waiting for stop event for 10 seconds'
)
univ.stop_event.wait(
10) #on failure, wait for some time
try:
os.remove(file_name) #remove the dump as we sent it
_log.info('Removed dump %s' % file_name)
except Exception as e:
_log.error('Failed to remove dump %s; %s' %
(file_name, unicode(e)))
if univ.stop_event.is_set(): #do we need to stop now
_log.debug('CrashDumpSender received stop event')
return
except:
pass
