def process_record(self, record):
#TODO: yield the value for processing to gratia ()
# logfile attribute (if present) is used to keep track and delete files
DebugPrint(5, "Creating JUR for %s" % record)
# Filter out uninteresting records (and remove their files)
if False:
if 'gratia_logfile' in record:
DebugPrint(
1, 'Deleting transient record file: ' +
record["gratia_logfile"])
file_utils.RemoveFile(record['gratia_logfile'])
raise IgnoreRecordException("Ignoring record.")
# Define the record
# UsageRecord is defined in https://twiki.opensciencegrid.org/bin/view/Accounting/ProbeDevelopement
# setters have the name of the attribute
# Set resource type ( Batch, BatchPilot, GridMonitor, Storage, ActiveTape )
resource_type = "Batch"
r = Gratia.UsageRecord(resource_type)
# fill r using the values in record
# remember to specify the transient file (that will be removed if the record
# is acquired successfully)
if 'logfile' in record:
r.AddTransientInputFile(record['gratia_logfile'])
return r
def removeCertInfoFile(xmlDoc, userIdentityNode, namespace):
''' Use localJobID and probeName to find cert info file and remove file'''
# Collect data needed by certinfo reader
DebugPrint(4, 'DEBUG: Get JobIdentity')
jobIdentityNode = GetNode(
xmlDoc.getElementsByTagNameNS(namespace, 'JobIdentity'))
if jobIdentityNode == None:
return
DebugPrint(4, 'DEBUG: Get JobIdentity: OK')
localJobId = GetNodeData(
jobIdentityNode.getElementsByTagNameNS(namespace, 'LocalJobId'))
DebugPrint(4, 'DEBUG: Get localJobId: ', localJobId)
usageRecord = userIdentityNode.parentNode
probeName = GetNodeData(
usageRecord.getElementsByTagNameNS(namespace, 'ProbeName'))
DebugPrint(4, 'DEBUG: Get probeName: ', probeName)
# Use _findCertinfoFile to find and remove the file, XML is ignored
# Looking only for exact match, globbing is disabled.
# Use _findCertinfoFile(localJobId, probeName) to look for more files with globbing if the exact matck
# is not found (gratia_certinfo_*_localJobId*)
DebugPrint(
4, 'DEBUG: call _findCertinfoFile(' + str(localJobId) + r', ' +
str(probeName) + ')')
certinfo_touple = _findCertinfoFile(localJobId, probeName)
if not certinfo_touple:
# matching certinfo file not found
DebugPrint(4, 'DEBUG: unable to find and remove certinfo file')
return None
# Get results and remove file
certinfo_fname = certinfo_touple[0]
DebugPrint(4, 'DEBUG: removing certinfo file' + str(certinfo_fname))
file_utils.RemoveFile(certinfo_fname) # Clean up.
return certinfo_fname
def classadToCertinfo(filename, output_dir):
"""
Process the classad of the finished job into a certinfo file in the same
directory. On failure, do not throw an exception, but quarantine the
classad file.
On success, the classad history file is deleted.
This function should not throw and does not return anything
"""
DebugPrint(4, "Converting ClassAd %s to certinfo file." % filename)
try:
fd = open(filename)
except IOError as ie:
DebugPrint(1, "Unable to open ClassAd %s for certinfo conversion" \
": %s" % (filename, str(ie)))
return
for classad in fdToClassad(fd):
if not createCertinfoFile(classad, output_dir):
DebugPrint(0, "Failed to convert certinfo file %s; sending to " \
"quarantine." % filename)
sandbox_mgmt.QuarantineFile(filename, False)
continue
file_utils.RemoveFile(filename)
def finalize_record(self, record_id):
"""Delete the file containing the record (the file name is passed as record_id)
:param record_id: file name of the record
:return: True if the record is found and the action performed
"""
if os.path.isfile(record_id):
file_utils.RemoveFile(record_id)
return True
return False
def readCertInfoFile(localJobId, probeName):
''' Look for and read contents of cert info file if present. And delete the file'''
certinfo_touple = _findCertinfoFile(localJobId, probeName)
if not certinfo_touple:
# matching certinfo file not found
return None
# Get results and remove file
result = {
'DN': GetNodeData(certinfo_touple[1].getElementsByTagName('DN'), 0),
'VO': GetNodeData(certinfo_touple[1].getElementsByTagName('VO'), 0),
'FQAN': GetNodeData(certinfo_touple[1].getElementsByTagName('FQAN'), 0)
}
DebugPrint(4, 'readCertInfo: removing ' + str(certinfo_touple[0]))
file_utils.RemoveFile(certinfo_touple[0]) # Clean up.
return result
def process_record(self, record):
#TODO: yield the value for processing to gratia ()
# logfile attribute (if present) is used to keep track and delete files
DebugPrint(5, "Creating JUR for %s" % record)
# Filter out uninteresting records (and remove their files)
if False:
if 'gratia_logfile' in record:
DebugPrint(
1, 'Deleting transient record file: ' +
record["gratia_logfile"])
file_utils.RemoveFile(record['gratia_logfile'])
raise IgnoreRecordException("Ignoring record.")
# Define the record
# UsageRecord is defined in https://twiki.opensciencegrid.org/bin/view/Accounting/ProbeDevelopement
# setters have the name of the attribute
# Set resource type ( Batch, BatchPilot, GridMonitor, Storage, ActiveTape )
resource_type = "Batch"
r = Gratia.UsageRecord(resource_type)
# fill r using the values in record
# remember to specify the transient file (that will be removed if the record
# is acquired successfully)
if 'logfile' in record:
r.AddTransientInputFile(record['gratia_logfile'])
return r
# TODO: end of part to remove
#############################################################
# Some references
# http://seann.herdejurgen.com/resume/samag.com/html/v11/i04/a6.htm
# http://stackoverflow.com/questions/14863224/efficient-reading-of-800-gb-xml-file-in-python-2-7
# http://radimrehurek.com/2014/03/data-streaming-in-python-generators-iterators-iterables/
try:
fd = open(filename)
except IOError, ie:
DebugPrint(1, "Unable to open ClassAd %s for certinfo conversion" \
": %s" % (filename, str(ie)))
return
for classad in fdToClassad(fd):
if not createCertinfoFile(classad, output_dir):
DebugPrint(0, "Failed to convert certinfo file %s; sending to " \
"quarantine." % filename)
sandbox_mgmt.QuarantineFile(filename, False)
continue
file_utils.RemoveFile(filename)
historyRe = re.compile("history\.[0-9]+\.[0-9]+")
def processHistoryDir():
"""
Condor schedd will write one file per finished job into this directory.
We must convert it from a Condor ClassAd to a certinfo file and rename it
based on the routed job's name.
"""
history_dir = Config.get_CondorCEHistoryFolder()
output_dir = Config.get_DataFolder()
if not history_dir:
DebugPrint(3, "No Condor-CE history specified; will not process for" \
