def extract_job_info(self, run_dir, kickstart_output):
"""
This function reads the output from the kickstart parser and
extracts the job information for the Stampede schema. It first
looks for an invocation record, and then for a clustered
record.
Returns None if an error occurs, True if an invocation record
was found, and False if it wasn't.
"""
# Check if we have anything
if len(kickstart_output) == 0:
return None
# Kickstart was parsed
self._kickstart_parsed = True
# Let's try to find an invocation record...
my_invocation_found = False
my_task_number = 0
self._stdout_text = "" # Initialize stdout
stdout_text_list = []
stdout_size=0
for my_record in kickstart_output:
if not "invocation" in my_record:
# Not this one... skip to the next
continue
# Ok, we have an invocation record, extract the information we
# need. Note that this may overwrite information obtained from
# the submit file (e.g. the site_name).
# Increment task_number
my_task_number = my_task_number + 1
if not my_invocation_found:
# Things we only need to do once
if "resource" in my_record:
self._site_name = my_record["resource"]
if "user" in my_record:
self._remote_user = my_record["user"]
if "cwd" in my_record:
self._remote_working_dir = my_record["cwd"]
if "hostname" in my_record:
self._host_id = my_record["hostname"]
# We are done with this part
my_invocation_found = True
#PM-641 optimization Modified string concatenation to a list join
if "stdout" in my_record:
if len(my_record["stdout"])<= MAX_OUTPUT_LENGTH - stdout_size:
stdout_text_list.append(utils.quote("#@ %d stdout\n" % (my_task_number)))
stdout_text_list.append(utils.quote(my_record["stdout"]))
stdout_text_list.append(utils.quote("\n"))
stdout_size+=len(my_record["stdout"])+20
if "stderr" in my_record:
if len(my_record["stderr"]) <= MAX_OUTPUT_LENGTH - stdout_size :
stdout_text_list.append(utils.quote("#@ %d stderr\n" % (my_task_number)))
stdout_text_list.append(utils.quote(my_record["stderr"]))
stdout_text_list.append(utils.quote("\n"))
stdout_size+=len(my_record["stderr"])+20
if len(stdout_text_list) > 0 :
self._stdout_text = "".join(stdout_text_list)
#PM-641 optimization merged encoding above
# Now, we encode it!
# if self._stdout_text != "":
# self._stdout_text = utils.quote(self._stdout_text)
if not my_invocation_found:
logger.debug("cannot find invocation record in output")
# Look for clustered record...
my_cluster_found = False
for my_record in kickstart_output:
if not "clustered" in my_record:
# Not this one... skip to the next
continue
# Ok found it, fill in cluster parameters
if "duration" in my_record:
self._cluster_duration = my_record["duration"]
if "start" in my_record:
# Convert timestamp to EPOCH
my_start = utils.epochdate(my_record["start"])
if my_start is not None:
self._cluster_start_time = my_start
# No need to look further...
my_cluster_found = True
break
if not my_cluster_found:
logger.debug("cannot find cluster record in output")
# Finally, read error file only
my_err_file = os.path.join(run_dir, self._error_file)
if my_invocation_found:
# in my job output there were some invocation records
# assume then that they are rotated also
my_err_file = my_err_file + ".%03d" % (self._job_output_counter)
try:
ERR = open(my_err_file, 'r')
self._stderr_text = utils.quote(ERR.read())
except IOError:
self._stderr_text = None
logger.warning("unable to read error file: %s, continuing..." % (my_err_file))
else:
ERR.close()
# Done populating Job class with information from the output file
return my_invocation_found
def extract_job_info(self, run_dir, kickstart_output):
"""
This function reads the output from the kickstart parser and
extracts the job information for the Stampede schema. It first
looks for an invocation record, and then for a clustered
record.
Returns None if an error occurs, True if an invocation record
was found, and False if it wasn't.
"""
# Check if we have anything
if len(kickstart_output) == 0:
return None
# Kickstart was parsed
self._kickstart_parsed = True
# Let's try to find an invocation record...
my_invocation_found = False
my_task_number = 0
self._stdout_text = "" # Initialize stdout
stdout_text_list = []
stdout_size=0
for my_record in kickstart_output:
if not "invocation" in my_record:
# Not this one... skip to the next
continue
# Ok, we have an invocation record, extract the information we
# need. Note that this may overwrite information obtained from
# the submit file (e.g. the site_name).
# Increment task_number
my_task_number = my_task_number + 1
if not my_invocation_found:
# Things we only need to do once
if "resource" in my_record:
self._site_name = my_record["resource"]
if "user" in my_record:
self._remote_user = my_record["user"]
if "cwd" in my_record:
self._remote_working_dir = my_record["cwd"]
if "hostname" in my_record:
self._host_id = my_record["hostname"]
# We are done with this part
my_invocation_found = True
#PM-641 optimization Modified string concatenation to a list join
if "stdout" in my_record:
if len(my_record["stdout"])<= MAX_OUTPUT_LENGTH - stdout_size:
try:
stdout_text_list.append(utils.quote("#@ %d stdout\n" % (my_task_number)))
stdout_text_list.append(utils.quote(my_record["stdout"]))
stdout_text_list.append(utils.quote("\n"))
stdout_size+=len(my_record["stdout"])+20
except KeyError:
logger.exception( "Unable to parse stdout section from kickstart record for task %s from file %s " %(my_task_number, self.get_rotated_out_filename() ))
if "stderr" in my_record:
if len(my_record["stderr"]) <= MAX_OUTPUT_LENGTH - stdout_size :
try:
stdout_text_list.append(utils.quote("#@ %d stderr\n" % (my_task_number)))
stdout_text_list.append(utils.quote(my_record["stderr"]))
stdout_text_list.append(utils.quote("\n"))
stdout_size+=len(my_record["stderr"])+20
except KeyError:
logger.exception( "Unable to parse stderr section from kickstart record for task %s from file %s " %(my_task_number, self.get_rotated_out_filename() ))
if len(stdout_text_list) > 0 :
self._stdout_text = "".join(stdout_text_list)
#PM-641 optimization merged encoding above
# Now, we encode it!
# if self._stdout_text != "":
# self._stdout_text = utils.quote(self._stdout_text)
if not my_invocation_found:
logger.debug("cannot find invocation record in output")
# Look for clustered record...
my_cluster_found = False
for my_record in kickstart_output:
if not "clustered" in my_record:
# Not this one... skip to the next
continue
# Ok found it, fill in cluster parameters
if "duration" in my_record:
self._cluster_duration = my_record["duration"]
if "start" in my_record:
# Convert timestamp to EPOCH
my_start = utils.epochdate(my_record["start"])
if my_start is not None:
self._cluster_start_time = my_start
# No need to look further...
my_cluster_found = True
break
if not my_cluster_found:
logger.debug("cannot find cluster record in output")
# Finally, read error file only
my_err_file = os.path.join(run_dir, self._error_file)
if my_invocation_found:
# in my job output there were some invocation records
# assume then that they are rotated also
my_err_file = my_err_file + ".%03d" % (self._job_output_counter)
try:
ERR = open(my_err_file, 'r')
self._stderr_text = utils.quote(ERR.read())
except IOError:
self._stderr_text = None
logger.warning("unable to read error file: %s, continuing..." % (my_err_file))
else:
ERR.close()
# Done populating Job class with information from the output file
return my_invocation_found
def extract_job_info(self, kickstart_output):
"""
This function reads the output from the kickstart parser and
extracts the job information for the Stampede schema. It first
looks for an invocation record, and then for a clustered
record.
Returns None if an error occurs, True if an invocation record
was found, and False if it wasn't.
"""
# Check if we have anything
if len(kickstart_output) == 0:
return None
# Kickstart was parsed
self._kickstart_parsed = True
# PM-1157 we construct run dir from job submit dir
run_dir = self._job_submit_dir
# Let's try to find an invocation record...
my_invocation_found = False
my_task_number = 0
self._stdout_text = "" # Initialize stdout
stdout_text_list = []
stdout_size=0
for my_record in kickstart_output:
if not "invocation" in my_record:
# Not this one... skip to the next
continue
# Ok, we have an invocation record, extract the information we
# need. Note that this may overwrite information obtained from
# the submit file (e.g. the site_name).
# Increment task_number
my_task_number = my_task_number + 1
if not my_invocation_found:
# Things we only need to do once
if "resource" in my_record:
self._site_name = my_record["resource"]
if "user" in my_record:
self._remote_user = my_record["user"]
if "cwd" in my_record:
self._remote_working_dir = my_record["cwd"]
if "hostname" in my_record:
self._host_id = my_record["hostname"]
# We are done with this part
my_invocation_found = True
# PM-1109 encode signal information if it exists
signal_message = " "
if "signalled" in my_record:
# construct our own error message
attrs = my_record["signalled"]
signal_message = "Job was "
if "action" in attrs:
signal_message += attrs["action"]
if "signal" in attrs:
signal_message += " with signal " + attrs["signal"]
#PM-641 optimization Modified string concatenation to a list join
if "stdout" in my_record:
task_output = self.split_task_output( my_record["stdout"])
self._add_additional_monitoring_events(task_output.events)
# PM-1152 we always attempt to store upto MAX_OUTPUT_LENGTH
stdout = self.get_snippet_to_populate( task_output.user_data, my_task_number, stdout_size, "stdout")
if stdout is not None:
try:
stdout_text_list.append(utils.quote("#@ %d stdout\n" % (my_task_number)))
stdout_text_list.append(utils.quote(stdout))
stdout_text_list.append(utils.quote("\n"))
stdout_size += len(stdout) + 20
except KeyError:
logger.exception( "Unable to parse stdout section from kickstart record for task %s from file %s " %(my_task_number, self.get_rotated_out_filename() ))
if "stderr" in my_record:
task_error = self.split_task_output(my_record["stderr"])
# add the events to those retrieved from the application stderr
self._add_additional_monitoring_events(task_error.events)
# Note: we are populating task stderr from kickstart record to job stdout only
stderr = self.get_snippet_to_populate( signal_message + task_error.user_data, my_task_number, stdout_size, "stderr")
if stderr is not None:
try:
stdout_text_list.append(utils.quote("#@ %d stderr\n" % (my_task_number)))
stdout_text_list.append(utils.quote(stderr))
stdout_text_list.append(utils.quote("\n"))
stdout_size += len( stderr ) + 20
except KeyError:
logger.exception( "Unable to parse stderr section from kickstart record for task %s from file %s " %(my_task_number, self.get_rotated_out_filename() ))
if len(stdout_text_list) > 0 :
self._stdout_text = "".join(stdout_text_list)
#PM-641 optimization merged encoding above
# Now, we encode it!
# if self._stdout_text != "":
# self._stdout_text = utils.quote(self._stdout_text)
if not my_invocation_found:
logger.debug("cannot find invocation record in output")
# Look for clustered record...
my_cluster_found = False
for my_record in kickstart_output:
if not "clustered" in my_record:
# Not this one... skip to the next
continue
# Ok found it, fill in cluster parameters
if "duration" in my_record:
self._cluster_duration = my_record["duration"]
if "start" in my_record:
# Convert timestamp to EPOCH
my_start = utils.epochdate(my_record["start"])
if my_start is not None:
self._cluster_start_time = my_start
# No need to look further...
my_cluster_found = True
break
if not my_cluster_found:
logger.debug("cannot find cluster record in output")
# Done populating Job class with information from the output file
return my_invocation_found
def extract_job_info(self, kickstart_output):
"""
This function reads the output from the kickstart parser and
extracts the job information for the Stampede schema. It first
looks for an invocation record, and then for a clustered
record.
Returns None if an error occurs, True if an invocation record
was found, and False if it wasn't.
"""
# Check if we have anything
if len(kickstart_output) == 0:
return None
# Kickstart was parsed
self._kickstart_parsed = True
# PM-1157 we construct run dir from job submit dir
run_dir = self._job_submit_dir
# Let's try to find an invocation record...
my_invocation_found = False
my_task_number = 0
self._stdout_text = "" # Initialize stdout
stdout_text_list = []
stdout_size = 0
for my_record in kickstart_output:
if not "invocation" in my_record:
# Not this one... skip to the next
continue
# Ok, we have an invocation record, extract the information we
# need. Note that this may overwrite information obtained from
# the submit file (e.g. the site_name).
# Increment task_number
my_task_number = my_task_number + 1
if not my_invocation_found:
# Things we only need to do once
if "resource" in my_record:
self._site_name = my_record["resource"]
if "user" in my_record:
self._remote_user = my_record["user"]
if "cwd" in my_record:
self._remote_working_dir = my_record["cwd"]
if "hostname" in my_record:
self._host_id = my_record["hostname"]
# We are done with this part
my_invocation_found = True
# PM-1109 encode signal information if it exists
signal_message = " "
if "signalled" in my_record:
# construct our own error message
attrs = my_record["signalled"]
signal_message = "Job was "
if "action" in attrs:
signal_message += attrs["action"]
if "signal" in attrs:
signal_message += " with signal " + attrs["signal"]
#PM-641 optimization Modified string concatenation to a list join
if "stdout" in my_record:
# PM-1152 we always attempt to store upto MAX_OUTPUT_LENGTH
stdout = self.get_snippet_to_populate(my_record["stdout"],
my_task_number,
stdout_size, "stdout")
if stdout is not None:
try:
stdout_text_list.append(
utils.quote("#@ %d stdout\n" % (my_task_number)))
stdout_text_list.append(utils.quote(stdout))
stdout_text_list.append(utils.quote("\n"))
stdout_size += len(stdout) + 20
except KeyError:
logger.exception(
"Unable to parse stdout section from kickstart record for task %s from file %s "
%
(my_task_number, self.get_rotated_out_filename()))
if "stderr" in my_record:
# Note: we are populating task stderr from kickstart record to job stdout only
stderr = self.get_snippet_to_populate(
signal_message + my_record["stderr"], my_task_number,
stdout_size, "stderr")
if stderr is not None:
try:
stdout_text_list.append(
utils.quote("#@ %d stderr\n" % (my_task_number)))
stdout_text_list.append(utils.quote(stderr))
stdout_text_list.append(utils.quote("\n"))
stdout_size += len(stderr) + 20
except KeyError:
logger.exception(
"Unable to parse stderr section from kickstart record for task %s from file %s "
%
(my_task_number, self.get_rotated_out_filename()))
if len(stdout_text_list) > 0:
self._stdout_text = "".join(stdout_text_list)
#PM-641 optimization merged encoding above
# Now, we encode it!
# if self._stdout_text != "":
# self._stdout_text = utils.quote(self._stdout_text)
if not my_invocation_found:
logger.debug("cannot find invocation record in output")
# Look for clustered record...
my_cluster_found = False
for my_record in kickstart_output:
if not "clustered" in my_record:
# Not this one... skip to the next
continue
# Ok found it, fill in cluster parameters
if "duration" in my_record:
self._cluster_duration = my_record["duration"]
if "start" in my_record:
# Convert timestamp to EPOCH
my_start = utils.epochdate(my_record["start"])
if my_start is not None:
self._cluster_start_time = my_start
# No need to look further...
my_cluster_found = True
break
if not my_cluster_found:
logger.debug("cannot find cluster record in output")
# Finally, read error file only
#my_err_file = os.path.join(run_dir, self._error_file)
basename = self._exec_job_id + ".err"
my_err_file = os.path.join(run_dir, basename)
if my_invocation_found:
# in my job output there were some invocation records
# assume then that they are rotated also
my_err_file = my_err_file + ".%03d" % (self._job_output_counter)
try:
ERR = open(my_err_file, 'r')
self._stderr_text = utils.quote(ERR.read())
except IOError:
self._stderr_text = None
if not self.is_noop_job():
logger.warning("unable to read error file: %s, continuing..." %
(my_err_file))
else:
ERR.close()
# Done populating Job class with information from the output file
return my_invocation_found
def testShortLocal(self):
"Should be able to get the epoch from a short local isodate"
self.assertEqual(
self.now, utils.epochdate(utils.isodate(now=self.now, short=True))
)
def testShortUTC(self):
"Should eb able to get the epoch from a short UTC isodate"
self.assertEqual(
self.now, utils.epochdate(utils.isodate(now=self.now, utc=True, short=True))
)
def extract_job_info(self, kickstart_output):
"""
This function reads the output from the kickstart parser and
extracts the job information for the Stampede schema. It first
looks for an invocation record, and then for a clustered
record.
Returns None if an error occurs, True if an invocation record
was found, and False if it wasn't.
"""
# Check if we have anything
if len(kickstart_output) == 0:
return None
# Kickstart was parsed
self._kickstart_parsed = True
# PM-1157 we construct run dir from job submit dir
self._job_submit_dir
# Let's try to find an invocation record...
my_invocation_found = False
my_task_number = 0
self._stdout_text = "" # Initialize stdout
stdout_text_list = []
stdout_size = 0
for my_record in kickstart_output:
if "multipart" in my_record:
# PM-1390 convert to integrity metrics
logger.debug("Multipart record %s", my_record)
self._add_multipart_events([my_record])
elif not "invocation" in my_record:
# Not this one... skip to the next
logger.trace("Skipping %s", my_record)
continue
# Ok, we have an invocation record, extract the information we
# need. Note that this may overwrite information obtained from
# the submit file (e.g. the site_name).
# Increment task_number
my_task_number = my_task_number + 1
if not my_invocation_found:
# Things we only need to do once
if "resource" in my_record:
self._site_name = my_record["resource"]
if "user" in my_record:
self._remote_user = my_record["user"]
if "cwd" in my_record:
self._remote_working_dir = my_record["cwd"]
if "hostname" in my_record:
self._host_id = my_record["hostname"]
# We are done with this part
my_invocation_found = True
# PM-1109 encode signal information if it exists
signal_message = " "
if "signalled" in my_record:
# construct our own error message
attrs = my_record["signalled"]
signal_message = "Job was "
if "action" in attrs:
signal_message += attrs["action"]
if "signal" in attrs:
signal_message += " with signal " + attrs["signal"]
# PM-641 optimization Modified string concatenation to a list join
if "stdout" in my_record:
task_output = self.split_task_output(my_record["stdout"])
self._add_additional_monitoring_events(task_output.events)
# PM-1152 we always attempt to store upto MAX_OUTPUT_LENGTH
stdout = self.get_snippet_to_populate(task_output.user_data,
my_task_number,
stdout_size, "stdout")
if stdout is not None:
try:
stdout_text_list.append(
utils.quote("#@ %d stdout\n" % (my_task_number)))
stdout_text_list.append(utils.quote(stdout))
stdout_text_list.append(utils.quote("\n"))
stdout_size += len(stdout) + 20
except KeyError:
logger.exception(
"Unable to parse stdout section from kickstart record for task %s from file %s "
%
(my_task_number, self.get_rotated_out_filename()))
if "stderr" in my_record:
task_error = self.split_task_output(my_record["stderr"])
# add the events to those retrieved from the application stderr
self._add_additional_monitoring_events(task_error.events)
# Note: we are populating task stderr from kickstart record to job stdout only
stderr = self.get_snippet_to_populate(
signal_message + task_error.user_data,
my_task_number,
stdout_size,
"stderr",
)
if stderr is not None:
try:
stdout_text_list.append(
utils.quote("#@ %d stderr\n" % (my_task_number)))
stdout_text_list.append(utils.quote(stderr))
stdout_text_list.append(utils.quote("\n"))
stdout_size += len(stderr) + 20
except KeyError:
logger.exception(
"Unable to parse stderr section from kickstart record for task %s from file %s "
%
(my_task_number, self.get_rotated_out_filename()))
# PM-1398 pass cpu info
if "cpu" in my_record:
self._cpu_attribs = my_record["cpu"]
if len(stdout_text_list) > 0:
self._stdout_text = "".join(stdout_text_list)
# PM-641 optimization merged encoding above
# Now, we encode it!
# if self._stdout_text != "":
# self._stdout_text = utils.quote(self._stdout_text)
if not my_invocation_found:
logger.debug("cannot find invocation record in output")
# Look for clustered record...
my_cluster_found = False
for my_record in kickstart_output:
if not "clustered" in my_record:
# Not this one... skip to the next
continue
# Ok found it, fill in cluster parameters
if "duration" in my_record:
self._cluster_duration = my_record["duration"]
if "start" in my_record:
# Convert timestamp to EPOCH
my_start = utils.epochdate(my_record["start"])
if my_start is not None:
self._cluster_start_time = my_start
# No need to look further...
my_cluster_found = True
break
if not my_cluster_found:
logger.debug("cannot find cluster record in output")
# Done populating Job class with information from the output file
return my_invocation_found
def testLocal(self):
"Should be able to get the epoch from a local isodate"
self.assertEqual(self.now, utils.epochdate(utils.isodate(now=self.now)))
def testShortUTC(self):
"Should eb able to get the epoch from a short UTC isodate"
self.assertEquals(self.now, utils.epochdate(utils.isodate(now=self.now, utc=True, short=True)))
def testShortLocal(self):
"Should be able to get the epoch from a short local isodate"
self.assertEquals(self.now, utils.epochdate(utils.isodate(now=self.now, short=True)))
def testUTC(self):
"Should be able to get the epoch from a UTC isodate"
self.assertEqual(self.now, utils.epochdate(utils.isodate(now=self.now, utc=True)))
def testLocal(self):
"Should be able to get the epoch from a local isodate"
self.assertEqual(self.now, utils.epochdate(utils.isodate(now=self.now)))
def testUTC(self):
"Should be able to get the epoch from a UTC isodate"
self.assertEquals(
self.now, utils.epochdate(utils.isodate(now=self.now, utc=True)))
