def getXML(self, job, sitename, workdir, xmlstr=None, jr=False):
""" Get the metadata xml """
node_xml = ""
tolog("getXML called")
# for backwards compatibility
try:
experiment = job.experiment
except:
experiment = "unknown"
# do not send xml for state 'holding' (will be sent by a later pilot during job recovery)
if job.result[0] == 'holding' and sitename != "CERNVM":
pass
else:
# only create and send log xml if the log was transferred
if job.result[0] == 'failed' and isLogfileCopied(workdir):
# generate the xml string for log file
# at this time the job.workdir might have been removed (because this function can be called
# after the removal of workdir is done), so we make a new dir
xmldir = "%s/XML4PandaJob_%s" % (workdir, job.jobId)
# group rw permission added as requested by LYON
ec, rv = getstatusoutput("mkdir -m g+rw %s" % (xmldir))
if ec != 0:
tolog("!!WARNING!!1300!! Could not create xmldir from updatePandaServer: %d, %s (resetting to site workdir)" % (ec, rv))
cmd = "ls -l %s" % (xmldir)
out = getoutput(cmd)
tolog("%s \n%s" % (cmd, out))
xmldir = workdir
# which checksum command should be used? query the site mover
from SiteMoverFarm import getSiteMover
sitemover = getSiteMover(readpar('copytool'), "")
if os.environ.has_key('Nordugrid_pilot'):
fname = os.path.join(self.__pilot_initdir, job.logFile)
else:
fname = os.path.join(workdir, job.logFile)
if os.path.exists(fname):
fnamelog = "%s/logfile.xml" % (xmldir)
guids_status = PFCxml(experiment, fnamelog, fntag="lfn", alog=job.logFile, alogguid=job.tarFileGuid, jr=jr)
from SiteMover import SiteMover
ec, pilotErrorDiag, _fsize, _checksum = SiteMover.getLocalFileInfo(fname, csumtype=sitemover.getChecksumCommand())
if ec != 0:
tolog("!!WARNING!!1300!! getLocalFileInfo failed: (%d, %s, %s)" % (ec, str(_fsize), str(_checksum)))
tolog("!!WARNING!!1300!! Can not set XML (will not be sent to server)")
node_xml = ''
else:
ec, _strXML = updateMetadata(fnamelog, _fsize, _checksum)
if ec == 0:
tolog("Added (%s, %s) to metadata file (%s)" % (_fsize, _checksum, fnamelog))
else:
tolog("!!WARNING!!1300!! Could not add (%s, %s) to metadata file (%s). XML will be incomplete: %d" %\
(_fsize, _checksum, fnamelog, ec))
# add skipped file info
_skippedfname = os.path.join(workdir, "skipped.xml")
if os.path.exists(_skippedfname):
ec = addSkippedToPFC(fnamelog, _skippedfname)
try:
f = open(fnamelog)
except Exception,e:
tolog("!!WARNING!!1300!! Exception caught: Can not open the file %s: %s (will not send XML)" %\
(fnamelog, str(e)))
node_xml = ''
else:
node_xml = ''
for line in f:
node_xml += line
f.close()
# transfer logfile.xml to pilot init dir for Nordugrid
if os.environ.has_key('Nordugrid_pilot'):
try:
copy2(fnamelog, self.__pilot_initdir)
except Exception, e:
tolog("!!WARNING!!1600!! Exception caught: Could not copy NG log metadata file to init dir: %s" % str(e))
else:
tolog("Successfully copied NG log metadata file to pilot init dir: %s" % (self.__pilot_initdir))
def getXML(self, job, sitename, workdir, xmlstr=None, jr=False):
""" Get the metadata xml """
node_xml = ""
tolog("getXML called")
# for backwards compatibility
try:
experiment = job.experiment
except:
experiment = "unknown"
# do not send xml for state 'holding' (will be sent by a later pilot during job recovery)
if job.result[0] == 'holding' and sitename != "CERNVM":
pass
else:
# only create and send log xml if the log was transferred
if job.result[0] == 'failed' and isLogfileCopied(workdir):
# generate the xml string for log file
# at this time the job.workdir might have been removed (because this function can be called
# after the removal of workdir is done), so we make a new dir
xmldir = "%s/XML4PandaJob_%s" % (workdir, job.jobId)
# group rw permission added as requested by LYON
ec, rv = getstatusoutput("mkdir -m g+rw %s" % (xmldir))
if ec != 0:
tolog("!!WARNING!!1300!! Could not create xmldir from updatePandaServer: %d, %s (resetting to site workdir)" % (ec, rv))
cmd = "ls -l %s" % (xmldir)
out = getoutput(cmd)
tolog("%s \n%s" % (cmd, out))
xmldir = workdir
# which checksum command should be used? query the site mover
from SiteMoverFarm import getSiteMover
sitemover = getSiteMover(readpar('copytool'), "")
if os.environ.has_key('Nordugrid_pilot'):
fname = os.path.join(self.__pilot_initdir, job.logFile)
else:
fname = os.path.join(workdir, job.logFile)
if os.path.exists(fname):
fnamelog = "%s/logfile.xml" % (xmldir)
guids_status = PFCxml(experiment, fnamelog, fntag="lfn", alog=job.logFile, alogguid=job.tarFileGuid, jr=jr)
from SiteMover import SiteMover
ec, pilotErrorDiag, _fsize, _checksum = SiteMover.getLocalFileInfo(fname, csumtype=sitemover.getChecksumCommand())
if ec != 0:
tolog("!!WARNING!!1300!! getLocalFileInfo failed: (%d, %s, %s)" % (ec, str(_fsize), str(_checksum)))
tolog("!!WARNING!!1300!! Can not set XML (will not be sent to server)")
node_xml = ''
else:
ec, _strXML = updateMetadata(fnamelog, _fsize, _checksum)
if ec == 0:
tolog("Added (%s, %s) to metadata file (%s)" % (_fsize, _checksum, fnamelog))
else:
tolog("!!WARNING!!1300!! Could not add (%s, %s) to metadata file (%s). XML will be incomplete: %d" %\
(_fsize, _checksum, fnamelog, ec))
# add skipped file info
_skippedfname = os.path.join(workdir, "skipped.xml")
if os.path.exists(_skippedfname):
ec = addSkippedToPFC(fnamelog, _skippedfname)
try:
f = open(fnamelog)
except Exception,e:
tolog("!!WARNING!!1300!! Exception caught: Can not open the file %s: %s (will not send XML)" %\
(fnamelog, str(e)))
node_xml = ''
else:
node_xml = ''
for line in f:
node_xml += line
f.close()
# transfer logfile.xml to pilot init dir for Nordugrid
if os.environ.has_key('Nordugrid_pilot'):
try:
copy2(fnamelog, self.__pilot_initdir)
except Exception, e:
tolog("!!WARNING!!1600!! Exception caught: Could not copy NG log metadata file to init dir: %s" % str(e))
else:
tolog("Successfully copied NG log metadata file to pilot init dir: %s" % (self.__pilot_initdir))
node_xml = ''
tolog("!!WARNING!!1300!! XML not found, nothing to send to server")
else:
tolog("!!WARNING!!1300!! File %s does not exist and transfer lockfile not found (job from old pilot?)" % (fname))
node_xml = ''
elif xmlstr:
# xmlstr was set in postJobTask for all files
tolog("XML string set")
_skippedfname = os.path.join(workdir, "skipped.xml")
fname = "%s/metadata-%s.xml" % (workdir, job.jobId)
if os.path.exists(fname):
if os.path.exists(_skippedfname):
# add the skipped file info if needed
ec = addSkippedToPFC(fname, _skippedfname)
# transfer metadata to pilot init dir for Nordugrid
if os.environ.has_key('Nordugrid_pilot'):
try:
copy2(fname, self.__pilot_initdir)
except Exception, e:
tolog("!!WARNING!!1600!! Exception caught: Could not copy metadata file to init dir for NG: %s" % str(e))
else:
tolog("Successfully copied metadata file to pilot init dir for NG: %s" % (self.__pilot_initdir))
else:
tolog("Warning: Metadata does not exist: %s" % (fname))
tolog("Will send XML")
node_xml = xmlstr
node_xml = ''
tolog("!!WARNING!!1300!! XML not found, nothing to send to server")
else:
tolog("!!WARNING!!1300!! File %s does not exist and transfer lockfile not found (job from old pilot?)" % (fname))
node_xml = ''
elif xmlstr:
# xmlstr was set in postJobTask for all files
tolog("XML string set")
_skippedfname = os.path.join(workdir, "skipped.xml")
fname = "%s/metadata-%s.xml" % (workdir, job.jobId)
if os.path.exists(fname):
if os.path.exists(_skippedfname):
# add the skipped file info if needed
ec = addSkippedToPFC(fname, _skippedfname)
# transfer metadata to pilot init dir for Nordugrid
if os.environ.has_key('Nordugrid_pilot'):
try:
copy2(fname, self.__pilot_initdir)
except Exception, e:
tolog("!!WARNING!!1600!! Exception caught: Could not copy metadata file to init dir for NG: %s" % str(e))
else:
tolog("Successfully copied metadata file to pilot init dir for NG: %s" % (self.__pilot_initdir))
else:
tolog("Warning: Metadata does not exist: %s" % (fname))
tolog("Will send XML")
node_xml = xmlstr
