def job_stats(self): try: workflow = stampede_statistics.StampedeStatistics(self.__get_wf_db_url(), False) workflow.initialize(root_wf_id=self._wf_id) workflow.set_job_filter("all") job_retry_count_dict = {} content = [] for job in workflow.get_job_statistics(): kickstart = "0" if job.kickstart == None else float(job.kickstart) multiplier_factor = "0" if job.multiplier_factor == None else int(job.multiplier_factor) kickstart_multi = "0" if job.kickstart_multi == None else float(job.kickstart_multi) remote_cpu_time = "0" if job.remote_cpu_time == None else float(job.remote_cpu_time) post_time = "0" if job.post_time == None else float(job.post_time) condor_q_time = "0" if job.condor_q_time == None else float(job.condor_q_time) resource_delay = "0" if job.resource_delay == None else float(job.resource_delay) runtime = "0" if job.runtime == None else float(job.runtime) seqexec = "-" if job.seqexec == None else float(job.seqexec) seqexec_delay = "-" if job.seqexec is not None and job.kickstart is not None: seqexec_delay = float(job.seqexec) - float(job.kickstart) if job_retry_count_dict.has_key(job.job_name): job_retry_count_dict[job.job_name] += 1 else: job_retry_count_dict[job.job_name] = 1 retry_count = job_retry_count_dict[job.job_name] content.append( [ job.job_name, retry_count, job.site, kickstart, multiplier_factor, kickstart_multi, remote_cpu_time, post_time, condor_q_time, resource_delay, runtime, seqexec, seqexec_delay, utils.raw_to_regular(job.exit_code), job.host_name, ] ) return content finally: Dashboard.close(workflow)
def job_stats(self): try: workflow = stampede_statistics.StampedeStatistics( self.__get_wf_db_url(), False) workflow.initialize(root_wf_id=self._wf_id) workflow.set_job_filter('all') job_retry_count_dict = {} content = [] for job in workflow.get_job_statistics(): kickstart = '0' if job.kickstart == None else float( job.kickstart) multiplier_factor = '0' if job.multiplier_factor == None else int( job.multiplier_factor) kickstart_multi = '0' if job.kickstart_multi == None else float( job.kickstart_multi) remote_cpu_time = '0' if job.remote_cpu_time == None else float( job.remote_cpu_time) post_time = '0' if job.post_time == None else float( job.post_time) condor_q_time = '0' if job.condor_q_time == None else float( job.condor_q_time) resource_delay = '0' if job.resource_delay == None else float( job.resource_delay) runtime = '0' if job.runtime == None else float(job.runtime) seqexec = '-' if job.seqexec == None else float(job.seqexec) seqexec_delay = '-' if job.seqexec is not None and job.kickstart is not None: seqexec_delay = (float(job.seqexec) - float(job.kickstart)) if job_retry_count_dict.has_key(job.job_name): job_retry_count_dict[job.job_name] += 1 else: job_retry_count_dict[job.job_name] = 1 retry_count = job_retry_count_dict[job.job_name] content.append([ job.job_name, retry_count, job.site, kickstart, multiplier_factor, kickstart_multi, remote_cpu_time, post_time, condor_q_time, resource_delay, runtime, seqexec, seqexec_delay, utils.raw_to_regular(job.exit_code), job.host_name ]) return content finally: Dashboard.close(workflow)
def createWorkflowEvent (self, record): ''' Map a result set into an event object. ''' event = WorkflowEvent () event.name = record.exec_job_id event.is_dax = record.exec_job_id.find ('subdax_') == 0 event.site = record.site event.timestamp = record.timestamp event.exitcode = utils.raw_to_regular (record.exitcode) event.hostname = record.hostname event.sched_id = record.sched_id event.work_dir = os.path.abspath (record.work_dir) event.work_dir = event.work_dir.replace ("work/outputs", "work") event.dax_file = os.path.basename (record.dax_file) #os.path.basename (record.dax_file) if record.dax_file else StatsUtil.formDaxName (event.workdir) event.state = record.state event.work_dir = record.work_dir event.stdout = record.stdout_file event.stderr = record.stderr_file #logger.debug (json.dumps (event, sort_keys=True, indent=3, cls=WorkflowEventEncoder)) return event '''
def process_invocation_notifications(self, wf, job, task_id, record=None): """ This function takes care of processing invocation-level notifications. """ if record is None: record = {} # Check if we have notifications for this workflow if not wf._wf_uuid in self._notifications: return # Get the notifications' dictionary for this workflow id wf_notifications = self._notifications[wf._wf_uuid] if "invocation" in wf_notifications: my_dict = wf_notifications["invocation"] else: logger.warning( "notification structure missing invocation entry...") return # Check if we have notifications for this job if not job._exec_job_id in my_dict: return # Advance to the task dictionary my_dict = my_dict[job._exec_job_id] # Check if we have notifications for this invocation if not task_id in my_dict: return my_notifications = my_dict[task_id] # Now, match the invocation state to the condition in the notification for k in my_notifications: # Look up the actions for this notification now my_actions = my_notifications[k] if "raw" in record: my_status = record["raw"] else: my_status = job._main_job_exitcode # Convert exitcode to int try: my_status = int(my_status) except ValueError: pass # Now, compare to the notification condition(s) if my_status == 0: if k == "on_error": continue if my_status != 0: if k == "on_success": continue if k == "all": k = "at_end" # Here, we always use the rotated file names as the invocation has already finished... my_output = os.path.join( wf._original_submit_dir, job._output_file) + ".%03d" % (job._job_output_counter) my_error = os.path.join( wf._original_submit_dir, job._error_file) + ".%03d" % (job._job_output_counter) # Ok, we have a match! for action in my_actions: # Create dictionary with needed environment variables my_env = {} my_env["PEGASUS_EVENT"] = k my_env["PEGASUS_EVENT_TIMESTAMP"] = str(wf._current_timestamp) my_env["PEGASUS_EVENT_TIMESTAMP_ISO"] = utils.isodate( wf._current_timestamp) my_env["PEGASUS_SUBMIT_DIR"] = wf._original_submit_dir my_env["PEGASUS_JOBID"] = job._exec_job_id my_env["PEGASUS_INVID"] = str(task_id) my_env["PEGASUS_WFID"] = ((wf._dax_label or "unknown") + "-" + (wf._dax_index or "unknown")) my_env["PEGASUS_STDOUT"] = my_output my_env["PEGASUS_STDERR"] = my_error if k != "start": # Convert raw exitcode into human-parseable format my_env["PEGASUS_STATUS"] = str( utils.raw_to_regular(my_status)) # Done, queue the notification self._pending_notifications.append((action, my_env))
def process_invocation_notifications(self, wf, job, task_id, record=None): """ This function takes care of processing invocation-level notifications. """ if record is None: record = {} # Check if we have notifications for this workflow if not wf._wf_uuid in self._notifications: return # Get the notifications' dictionary for this workflow id wf_notifications = self._notifications[wf._wf_uuid] if "invocation" in wf_notifications: my_dict = wf_notifications["invocation"] else: logger.warning("notification structure missing invocation entry...") return # Check if we have notifications for this job if not job._exec_job_id in my_dict: return # Advance to the task dictionary my_dict = my_dict[job._exec_job_id] # Check if we have notifications for this invocation if not task_id in my_dict: return my_notifications = my_dict[task_id] # Now, match the invocation state to the condition in the notification for k in my_notifications: # Look up the actions for this notification now my_actions = my_notifications[k] if "raw" in record: my_status = record["raw"] else: my_status = job._main_job_exitcode # Convert exitcode to int try: my_status = int(my_status) except ValueError: pass # Now, compare to the notification condition(s) if my_status == 0: if k == "on_error": continue if my_status != 0: if k == "on_success": continue if k == "all": k = "at_end" # Here, we always use the rotated file names as the invocation has already finished... my_output = os.path.join(wf._original_submit_dir, job._output_file) + ".%03d" % (job._job_output_counter) my_error = os.path.join(wf._original_submit_dir, job._error_file) + ".%03d" % (job._job_output_counter) # Ok, we have a match! for action in my_actions: # Create dictionary with needed environment variables my_env = {} my_env["PEGASUS_EVENT"] = k my_env["PEGASUS_EVENT_TIMESTAMP"] = str(wf._current_timestamp) my_env["PEGASUS_EVENT_TIMESTAMP_ISO"] = utils.isodate(wf._current_timestamp) my_env["PEGASUS_SUBMIT_DIR"] = wf._original_submit_dir my_env["PEGASUS_JOBID"] = job._exec_job_id my_env["PEGASUS_INVID"] = str(task_id) my_env["PEGASUS_WFID"] = ((wf._dax_label or "unknown") + "-" + (wf._dax_index or "unknown")) my_env["PEGASUS_STDOUT"] = my_output my_env["PEGASUS_STDERR"] = my_error if k != "start": # Convert raw exitcode into human-parseable format my_env["PEGASUS_STATUS"] = str(utils.raw_to_regular(my_status)) # Done, queue the notification self._pending_notifications.append((action, my_env))