def execute_runbook(self): """Executes the job runtime and performs runtime operation (stream upload / status change).""" # set status to running tracer.log_sandbox_job_started(self.job_id, self.runbook.definition_kind_str, self.runbook_data.name, self.runbook_data.runbook_version_id) start_request_time = time.strptime(self.job_data.start_request_time.split("+")[0].split(".")[0], "%Y-%m-%dT%H:%M:%S") time_taken_to_start_td = datetime.utcnow() - datetime.fromtimestamp(time.mktime(start_request_time)) time_taken_to_start_in_seconds = (time_taken_to_start_td.microseconds + (time_taken_to_start_td.seconds + time_taken_to_start_td.days * 24 * 3600) * 10 ** 6) / 10 ** 6 tracer.log_etw_job_status_changed_running(self.job_data.subscription_id, self.job_data.account_id, self.job_data.account_name, self.sandbox_id, self.job_data.job_id, self.runbook.definition_kind_str, self.runbook_data.name, time_taken_to_start_in_seconds) tracer.log_etw_user_requested_start_or_resume(self.job_data.account_id, self.sandbox_id, self.job_data.job_id, self.runbook_data.name, self.job_data.account_name, time_taken_to_start_in_seconds, self.runbook.definition_kind_str) self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.RUNNING, False, self.get_job_extended_properties()) # create runbook subprocess self.runtime.start_runbook_subprocess() # monitor runbook output for streams stream_handler = StreamHandler(self.job_data, self.runtime.runbook_subprocess, self.jrds_client) stream_handler.daemon = True stream_handler.start() # wait for runbook execution to complete pending_action = None while stream_handler.isAlive() or self.runtime.runbook_subprocess.poll() is None: try: pending_action = self.msg_queue.get(block=False) tracer.log_sandbox_job_pending_action_detected(self.job_id, pending_action) if pending_action == pendingactions.STOP_ENUM_INDEX: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPING, False) self.runtime.kill_runbook_subprocess() break except Queue.Empty: pass time.sleep(0.2) # handle terminal state changes if pending_action == pendingactions.STOP_ENUM_INDEX: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPED, True, self.get_job_extended_properties()) tracer.log_etw_job_status_changed_stopped(self.job_data.subscription_id, self.job_data.account_id, self.job_data.account_name, self.sandbox_id, self.job_data.job_id, self.runbook.definition_kind_str, self.runbook_data.name) elif self.runtime.runbook_subprocess.poll() is not None and self.runtime.runbook_subprocess.poll() == EXIT_SUCCESS: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.COMPLETED, True, self.get_job_extended_properties()) tracer.log_etw_job_status_changed_completed(self.job_data.subscription_id, self.job_data.account_id, self.job_data.account_name, self.sandbox_id, self.job_data.job_id, self.runbook.definition_kind_str, self.runbook_data.name) else: full_error_output = self.get_full_stderr_content(self.runtime.runbook_subprocess.stderr) self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True, self.get_job_extended_properties(), exception=full_error_output) tracer.log_etw_job_status_changed_failed(self.job_data.subscription_id, self.job_data.account_id, self.job_data.account_name, self.sandbox_id, self.job_id, self.runbook.definition_kind_str, self.runbook_data.name, self.runbook_data.runbook_version_id, full_error_output)
def run(server_only=False): # set up logging: logging.basicConfig( filename=LOG_FILE_NAME, format='%(asctime)s - %(levelname)s:%(message)s', level=logging.DEBUG ) try: import setproctitle setproctitle.setproctitle('anagramatron') except ImportError: print("missing module: setproctitle") pass if server_only: hit_server.start_hit_server() else: hitserver = multiprocessing.Process(target=hit_server.start_hit_server) hitserver.daemon = True hitserver.start() anagram_finder = AnagramFinder() stats.clear_stats() while 1: print('top of run loop') logging.debug('top of run loop') try: print('starting stream handler') stream_handler = StreamHandler() stream_handler.start() for processed_tweet in stream_handler: anagram_finder.handle_input(processed_tweet) stats.update_console() except NeedsMaintenance: logging.debug('caught NeedsMaintenance exception') print('performing maintenance') stream_handler.close() anagram_finder.perform_maintenance() except KeyboardInterrupt: stream_handler.close() anagram_finder.close() return 0 except Exception as err: logging.error(sys.exc_info()) stream_handler.close() anagram_finder.close() TwitterHandler().send_message(str(err) + "\n" + datetime.today().isoformat()) raise
def execute_runbook(self): """Executes the job runtime and performs runtime operation (stream upload / status change).""" # set status to running tracer.log_debug_trace("Starting runbook.") self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.RUNNING, False) # create runbook subprocess self.runtime.start_runbook_subprocess() # monitor runbook output for streams stream_handler = StreamHandler(self.job_data, self.runtime.runbook_subprocess, self.jrds_client) stream_handler.daemon = True stream_handler.start() # wait for runbook execution to complete pending_action = None while stream_handler.isAlive( ) or self.runtime.runbook_subprocess.poll() is None: try: pending_action = self.msg_queue.get(block=False) tracer.log_debug_trace("Pending action detected. " + str(pending_action)) if pending_action == pendingactions.STOP_ENUM_INDEX: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPING, False) self.runtime.kill_runbook_subprocess() break except Queue.Empty: pass time.sleep(0.2) # handle terminal state changes if pending_action == pendingactions.STOP_ENUM_INDEX: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPED, True) tracer.log_debug_trace("Completed - Stopped") elif self.runtime.runbook_subprocess.poll( ) is not None and self.runtime.runbook_subprocess.poll( ) == EXIT_SUCCESS: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.COMPLETED, True) tracer.log_debug_trace("Completed - Without error") else: full_error_output = self.get_full_stderr_content( self.runtime.runbook_subprocess.stderr) self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True, exception=full_error_output) tracer.log_debug_trace("Completed - With error")
def execute_runbook(self): """Executes the job runtime and performs runtime operation (stream upload / status change).""" # set status to running tracer.log_debug_trace("Starting runbook.") self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.RUNNING, False) # create runbook subprocess self.runtime.start_runbook_subprocess() # monitor runbook output for streams stream_handler = StreamHandler(self.job_data, self.runtime.runbook_subprocess, self.jrds_client) stream_handler.daemon = True stream_handler.start() # wait for runbook execution to complete pending_action = None while stream_handler.isAlive() or self.runtime.runbook_subprocess.poll() is None: try: pending_action = self.msg_queue.get(block=False) tracer.log_debug_trace("Pending action detected. " + str(pending_action)) if pending_action == pendingactions.STOP_ENUM_INDEX: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPING, False) self.runtime.kill_runbook_subprocess() break except Queue.Empty: pass time.sleep(0.2) # handle terminal state changes if pending_action == pendingactions.STOP_ENUM_INDEX: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPED, True) tracer.log_debug_trace("Completed - Stopped") elif self.runtime.runbook_subprocess.poll() is not None and self.runtime.runbook_subprocess.poll() == EXIT_SUCCESS: self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.COMPLETED, True) tracer.log_debug_trace("Completed - Without error") else: full_error_output = self.get_full_stderr_content(self.runtime.runbook_subprocess.stderr) self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True, exception=full_error_output) tracer.log_debug_trace("Completed - With error")
import cPickle as pickle import time import sys from streamhandler import StreamHandler import anagramfunctions """a helper file for fetching & saving test data from the twitter stream""" if __name__ == "__main__": stream = StreamHandler() stream.start() count = 0 save_interval = 50000 tlist = [] try: for t in stream: if not t: continue tlist.append(t) count += 1 sys.stdout.write(str(count) + '\r') sys.stdout.flush() if count > save_interval: filename = "testdata/filt_%s.p" % time.strftime("%b%d%H%M") pickle.dump(tlist, open(filename, 'wb')) count = 0 tlist = [] finally: if count > 1000: