def test(self): if get_active_stack(tester) is not None: print( "An active stack should only exist when added explicitly" ) return False with duration(self, "a name") as context_object: active_stack = get_active_stack(self) # We should have an active stack in the context if active_stack is None: print( "In duration context the active stack should be added." ) return False if not get_child(active_stack, "active_stack").hasChildNodes(): print( "in the context the active_stack should at least contain one entry" ) return False # Now leave the context if get_child(active_stack, "active_stack").hasChildNodes(): print("After the context the active stack should be left") # There is stil an entry in the active stack return False return True
def test(self): if get_active_stack(tester) is not None: print "An active stack should only exist when added explicitly" return False with duration(self, "a name") as context_object: active_stack = get_active_stack(self) # We should have an active stack in the context if active_stack is None: print "In duration context the active stack should be added." return False if not get_child( active_stack, "active_stack").hasChildNodes(): print "in the context the active_stack should at least contain one entry" return False # Now leave the context if get_child( active_stack, "active_stack").hasChildNodes(): print "After the context the active stack should be left" # There is stil an entry in the active stack return False return True
def wrapper(*args, **argsw): """ Decorator construct, receives arguments to the decorated function """ # Get the calling object (first argument supplied to this decorator) calling_object = args[0] try: # call the actual function time_info_start = time.time() return_value = target(*args, **argsw) time_info_end = time.time() # Force exception on non zero output if return_value != 0: raise Exception("Non zero pipeline output") # Mail main dev on succesfull run stack = get_active_stack(calling_object) duration_recipe = str(time_info_end - time_info_start) if stack != None: stack.setAttribute("duration", duration_recipe) msg_string = stack.toprettyxml(encoding='ascii') else: msg_string = "duration: {0} \n "\ "No additional pipeline data available".format(duration_recipe ) _mail_msg_to( "pipeline_finished", "*****@*****.**", "pipeline finished: {0}: {1}".format( os.path.basename(calling_object.__file__), calling_object.inputs['job_name']), msg_string) except Exception, message: # Static list of mail to be send (could be made configurable, # but yeah temp mail functionality so...) mail_list = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] # get the active stack stack = get_active_stack(calling_object) active_stack_data = "" if stack != None: active_stack_data = stack.toprettyxml(encoding='ascii') # get the Obsid and pipeline name add to subjecy title subject = "Failed pipeline run {0}: {1}".format( os.path.basename(calling_object.__file__), calling_object.inputs['job_name']) # construct the message msg = "Error ({0}): {1} \n information: \n {2}".format( type(message), message, active_stack_data) # mail all recipients for entry in mail_list: _mail_msg_to("pipeline_error", entry, subject, msg) raise
def wrapper(*args, **argsw): """ Decorator construct, receives arguments to the decorated function """ # Get the calling object (first argument supplied to this decorator) calling_object = args[0] try: # call the actual function time_info_start = time.time() return_value = target(*args, **argsw) time_info_end = time.time() # Force exception on non zero output if return_value != 0: raise Exception("Non zero pipeline output") # Mail main dev on succesfull run stack = get_active_stack(calling_object) duration_recipe = str(time_info_end - time_info_start) if stack != None: stack.setAttribute( "duration", duration_recipe) simplyfied_pipeline_xml = strip_xml_to_master_details( stack, calling_object.logger) msg_string = simplyfied_pipeline_xml.toprettyxml(encoding='ascii') else: msg_string = "duration: {0} \n "\ "No additional pipeline data available".format(duration_recipe) except Exception, message: _send_mail_notification(calling_object, message) raise message
def test_add_child_to_active_stack_head(self): class a_class(object): def __init__(self): pass local_document = xml.Document() created_node = local_document.createElement("Tester") an_object = a_class() return_value = xmllogging.add_child_to_active_stack_head( an_object, created_node) self.assertTrue( return_value == None, "function should return None when adding child when no active stack is there " ) xmllogging.enter_active_stack(an_object, "test") # Add the chilf return_value = xmllogging.add_child_to_active_stack_head( an_object, created_node) # get the stack stack = xmllogging.get_active_stack(an_object) stack_text = stack.toxml() goal_text = """<active_stack Name="a_class" type="active_stack"><active_stack info="Contains functions not left with a return"><test><Tester/></test></active_stack></active_stack>""" # The node text should have a Tester node added xmllogging.exit_active_stack(an_object) self.assertEqual(stack_text, goal_text, "THe created xml structure is not correct")
def wrapper(*args, **argsw): """ Decorator construct, receives arguments to the decorated function """ # Get the calling object (first argument supplied to this decorator) calling_object = args[0] try: # call the actual function time_info_start = time.time() return_value = target(*args, **argsw) time_info_end = time.time() # Force exception on non zero output if return_value != 0: raise Exception("Non zero pipeline output") # Mail main dev on succesfull run stack = get_active_stack(calling_object) duration_recipe = str(time_info_end - time_info_start) if stack != None: stack.setAttribute("duration", duration_recipe) simplyfied_pipeline_xml = strip_xml_to_master_details( stack, calling_object.logger) msg_string = simplyfied_pipeline_xml.toprettyxml( encoding='ascii').decode('ascii') else: msg_string = "duration: {0} \n "\ "No additional pipeline data available".format(duration_recipe) except Exception as message: _send_mail_notification(calling_object, message) raise message # return the actual value of the function return return_value
def test_add_child_to_active_stack_head(self): class a_class(object): def __init__(self): pass local_document = xml.Document() created_node = local_document.createElement("Tester") an_object = a_class() return_value = xmllogging.add_child_to_active_stack_head(an_object, created_node) self.assertTrue(return_value == None, "function should return None when adding child when no active stack is there ") xmllogging.enter_active_stack(an_object, "test") # Add the chilf return_value = xmllogging.add_child_to_active_stack_head(an_object, created_node) # get the stack stack = xmllogging.get_active_stack(an_object) stack_text = stack.toxml() goal_text = """<active_stack Name="a_class" type="active_stack"><active_stack info="Contains functions not left with a return"><test><Tester/></test></active_stack></active_stack>""" # The node text should have a Tester node added xmllogging.exit_active_stack(an_object) self.assertEqual(stack_text, goal_text, "THe created xml structure is not correct")
def test_get_active_stack(self): class a_class(object): def __init__(self): pass an_object = a_class() result = xmllogging.get_active_stack(an_object) # If no active stack is created return None self.assertTrue(result == None, "When no active stack is entered" " get_active_stack should return None") xmllogging.enter_active_stack(an_object, "test", stack_name="test_stack") result = xmllogging.get_active_stack(an_object) # Calling get stack with incorrect name (default in this case) return None self.assertTrue(result == None, "When incorrect active stack name is entered" " get_active_stack should return None") xmllogging.exit_active_stack(an_object, stack_name="test_stack")
def _send_mail_notification(calling_object, message): # Static list of mail to be send mail_list = [ "*****@*****.**" ] calling_object.logger.info("creating email notification...") # get the active stack active_stack_data = '???' try: stack = get_active_stack(calling_object) if stack is not None: active_stack_data = stack simplyfied_pipeline_xml = strip_xml_to_master_details( stack, calling_object.logger) active_stack_data = simplyfied_pipeline_xml.toprettyxml( encoding='ascii') except Exception as e: calling_object.logger.error(e) # get the Obsid and pipeline name add to subjecy title obsid = '???' jobname = '???' try: obsid = os.path.basename(calling_object.__file__) jobname = calling_object.inputs['job_name'] except Exception as e: calling_object.logger.error(e) subject = "Failed pipeline run {0}: {1}".format(obsid, jobname) # construct the message msg = "Error ({0}): {1} \n information: \n {2}".format( type(message), message, active_stack_data) # mail all recipients try: pconfig = PipelineEmailConfig() error_sender = pconfig['error-sender'] # provoke_exception if key missing except Exception as e: calling_object.logger.error(e) error_sender = "*****@*****.**" calling_object.logger.warn("Could not find sender address. Using default: %s", error_sender) for entry in mail_list: calling_object.logger.info("sending email notification '%s' to %s", subject, entry) _mail_msg_to(error_sender, entry, subject, msg)
def test_get_active_stack(self): class a_class(object): def __init__(self): pass an_object = a_class() result = xmllogging.get_active_stack(an_object) # If no active stack is created return None self.assertTrue( result == None, "When no active stack is entered" " get_active_stack should return None") xmllogging.enter_active_stack(an_object, "test", stack_name="test_stack") result = xmllogging.get_active_stack(an_object) # Calling get stack with incorrect name (default in this case) return None self.assertTrue( result == None, "When incorrect active stack name is entered" " get_active_stack should return None") xmllogging.exit_active_stack(an_object, stack_name="test_stack")
def _send_mail_notification(calling_object, message): # Static list of mail to be send mail_list = ["*****@*****.**"] # get the active stack active_stack_data = '???' try: stack = get_active_stack(calling_object) if stack is not None: active_stack_data = stack simplyfied_pipeline_xml = strip_xml_to_master_details( stack, calling_object.logger) active_stack_data = simplyfied_pipeline_xml.toprettyxml( encoding='ascii').decode('ascii') except: pass # get the Obsid and pipeline name add to subjecy title obsid = '???' jobname = '???' try: obsid = os.path.basename(calling_object.__file__) jobname = calling_object.inputs['job_name'] except: pass subject = "Failed pipeline run {0}: {1}".format(obsid, jobname) # construct the message msg = "Error ({0}): {1} \n information: \n {2}".format( type(message), message, active_stack_data) # mail all recipients try: pconfig = PipelineEmailConfig() error_sender = pconfig[ 'error-sender'] # provoke_exception if key missing except Exception as e: print(e) # raise Exception("loggingdecorators.py: Could not find the pipeline email configuration file: %s" % (e) ) error_sender = "*****@*****.**" for entry in mail_list: _mail_msg_to(error_sender, entry, subject, msg)
def wrapper(*args, **argsw): """ Decorator construct, receives arguments to the decorated function """ # Get the calling object (first argument supplied to this decorator) calling_object = args[0] try: # call the actual function time_info_start = time.time() return_value = target(*args, **argsw) time_info_end = time.time() # Force exception on non zero output if return_value != 0: raise Exception("Non zero pipeline output") # Mail main dev on succesfull run stack = get_active_stack(calling_object) duration_recipe = str(time_info_end - time_info_start) if stack != None: stack.setAttribute( "duration", duration_recipe) simplyfied_pipeline_xml = strip_xml_to_master_details( stack, calling_object.logger) msg_string = simplyfied_pipeline_xml.toprettyxml(encoding='ascii') else: msg_string = "duration: {0} \n "\ "No additional pipeline data available".format(duration_recipe) _mail_msg_to("pipeline_finished", "*****@*****.**", "pipeline finished: {0}: {1}".format( os.path.basename(calling_object.__file__), calling_object.inputs['job_name']), msg_string) except Exception, message: # Static list of mail to be send mail_list = ["*****@*****.**", "*****@*****.**", "*****@*****.**" ] # get the active stack stack = get_active_stack(calling_object) active_stack_data = "" if stack != None: simplyfied_pipeline_xml = strip_xml_to_master_details( stack, calling_object.logger) active_stack_data = simplyfied_pipeline_xml.toprettyxml( encoding='ascii') # get the Obsid and pipeline name add to subjecy title subject = "Failed pipeline run {0}: {1}".format( os.path.basename(calling_object.__file__), calling_object.inputs['job_name']) # construct the message msg = "Error ({0}): {1} \n information: \n {2}".format( type(message), message, active_stack_data) # mail all recipients for entry in mail_list: _mail_msg_to("pipeline_error", entry, subject, msg) raise
def go(self): # Read the parset-file that was given as input argument try: self.parset_file = os.path.abspath(self.inputs['args'][0]) except IndexError: return self.usage() self.parset.adoptFile(self.parset_file) # Set job-name to basename of parset-file w/o extension, if it's not # set on the command-line with '-j' or '--job-name' if not 'job_name' in self.inputs: self.inputs['job_name'] = (os.path.splitext( os.path.basename(self.parset_file))[0]) # we can call our parent now that we have a job_name super(control, self).go() # we now have a self.config -- read our settings try: self.feedback_method = self.config.get('feedback', 'method') except: self.feedback_method = "messagebus" try: self.feedback_send_status = self.config.getboolean( 'feedback', 'send_status') except: self.feedback_send_status = True if self.feedback_method == "messagebus" and not messagebus.MESSAGING_ENABLED: self.logger.error( "Feedback over messagebus requested, but messagebus support is not enabled or functional" ) return 1 # Pull several parameters from the parset self.momID = self.parset.getString( "ObsSW.Observation.momID", "") # Note: 0 if obs was copied in Scheduler self.sasID = self.parset.getString("ObsSW.Observation.otdbID", "") # SAS ID # Start the pipeline self.logger.info("LOFAR Pipeline (%s) starting." % self.name) self.logger.info("SASID = %s, MOMID = %s, Feedback method = %s" % (self.sasID, self.momID, self.feedback_method)) try: self.pipeline_logic() except Exception as message: self.logger.error("*******************************************") self.logger.error("Failed pipeline run: {0}".format( self.inputs['job_name'])) # Get detailed information of the caught exception (type, value, traceback_object) = sys.exc_info() self.logger.error("Detailed exception information:") self.logger.error(str(type)) self.logger.error(str(value)) self.logger.exception(message) # Get the stacktrace and pretty print it: self.logger.error("\n" + " ".join( traceback.format_list(traceback.extract_tb(traceback_object)))) self.logger.error("*******************************************") # Emit process status self._send_feedback_status(1) self.logger.error("LOFAR Pipeline finished unsuccesfully.") return 1 else: # Emit process status self._send_feedback_status(0) self.logger.info("LOFAR Pipeline finished succesfully.") return 0 finally: # always print a xml stats file if get_active_stack(self) != None: xmlfile = self.config.get("logging", "xml_stat_file") try: fp = open(xmlfile, "w") fp.write( get_active_stack(self).toxml( encoding='ascii').decode('ascii')) fp.close() except Exception as except_object: self.logger.error("Failed opening xml stat file:") self.logger.error(except_object) return 0
# Get detailed information of the caught exception (type, value, traceback_object) = sys.exc_info() self.logger.error("Detailed exception information:") self.logger.error(str(type)) self.logger.error(str(value)) # Get the stacktrace and pretty print it: # self.logger.error("\n" + " ".join(traceback.format_list( # traceback.extract_tb(traceback_object)))) self.logger.error("*******************************************") self._send_mac_feedback(1) return 1 else: self._send_mac_feedback(0) return 0 finally: # always print a xml stats file if get_active_stack(self) != None: xmlfile = self.config.get("logging", "xml_stat_file") try: fp = open(xmlfile, "w") fp.write(get_active_stack(self).toxml(encoding='ascii')) fp.close() except Exception, except_object: self.logger.error("Failed opening xml stat file:") self.logger.error(except_object) return 0