def record_completion(self):
self.dbi.set_obs_status(self.obs, self.task)
self.dbi.set_obs_pid(self.obs, 0)
self.remove_file_if_exists(self.stdout_stderr_file)
if self.ts.wf.log_to_mc:
# log completion time to M&C
import mc_utils
mc_utils.add_mc_process_event(self.obs, "finished")
mc_utils.add_mc_process_record(self.obs,
self.ts.wf.workflow_actions,
self.ts.wf.workflow_actions_endfile)
# log task resources to M&C
stop_time = Time.now()
if self.max_mem > 0:
max_mem = self.max_mem
else:
max_mem = None
if self.n_cpu_load_polls > 0:
avg_cpu = self.avg_cpu_load / self.n_cpu_load_polls
else:
avg_cpu = None
mc_utils.add_mc_task_resource_record(self.obs, self.task,
self.start_time,
self.stop_time, max_mem,
avg_cpu)
return
def record_launch(self):
if self.sg.cluster_scheduler == 1:
self.dbi.set_obs_pid(self.obs, self.process)
else:
self.dbi.set_obs_pid(self.obs, self.process.pid)
# log to M&C
if self.ts.wf.log_to_mc:
import mc_utils
mc_utils.add_mc_process_event(self.obs, "started")
def record_failure(self, failure_type="FAILED"):
for task in self.ts.active_tasks:
if task.obs == self.obs:
# Remove the killed task from the active task list
self.ts.active_tasks.remove(task)
logger.debug("Removed task : %s from active list" % task.task)
self.dbi.set_obs_pid(self.obs, -9)
self.dbi.update_obs_current_stage(self.obs, failure_type)
logger.error("Task.record_failure: Task: %s, Obsnum: %s, Type: %s" %
(self.task, self.obs, failure_type))
if self.ts.wf.log_to_mc:
# log to M&C
import mc_utils
mc_utils.add_mc_process_event(self.obs, "error")
return
def test_add_mc_process_event(self):
# add observation to db
self.test_session.add_obs(*self.observation_values)
obs_result = self.test_session.get_obs()
nt.assert_equal(len(obs_result), 1)
# add process event to db
mc_utils.add_mc_process_event(*self.process_event,
mcs=self.test_session)
# retrieve record and check that it matches
result = self.test_session.get_rtp_process_event(
self.time - TimeDelta(2, format='sec'))
nt.assert_equal(len(result), 1)
# the time of the entry is not controlled by RTP, so don't check taht
result = result[0]
nt.assert_equal(result.obsid, self.process_event[0])
nt.assert_equal(result.event, self.process_event[1])
# also test dumping a dict to disk
# feeding in nonsense for an MCSession will trigger the "except" branch
mc_utils.add_mc_process_event(*self.process_event,
mcs='blah',
outdir=os.getcwd())
# read in pickle from disk
# there should only be one
for fn in os.listdir(os.getcwd()):
if 'pe_' in fn:
pe_fn = fn
break
with open(pe_fn, 'r') as f:
info_dict = pickle.load(f)
# check that dictionary entries match data
nt.assert_equal(info_dict['obsid'], self.process_event[0])
nt.assert_equal(info_dict['event'], self.process_event[1])
# clean up after ourselves
os.remove(pe_fn)
def add_observation(self,
obsnum,
date,
date_type,
pol,
filename,
host,
outputhost='',
length=10 / 60. / 24,
status='NEW',
path_prefix=None):
"""
create a new observation entry.
returns: obsnum (see jdpol2obsnum)
Note: does not link up neighbors!
"""
OBS = Observation(obsnum=obsnum,
date=str(date),
date_type=date_type,
pol=pol,
status=status,
outputhost=outputhost,
length=length)
s = self.Session()
s.add(OBS)
s.commit()
obsnum = OBS.obsnum
s.close()
self.add_file(obsnum, host, filename, path_prefix=path_prefix)
# add to m&c
try:
import mc_utils
except ImportError:
return obsnum
else:
mc_utils.add_mc_process_event(obsnum, 'queued')
return obsnum