def SendHandshake(record):
global successfulHandshakes
global failedHandshakes
DebugPrint(0,
'***********************************************************')
# Assemble the record into xml
record.XmlCreate()
# Parse it into nodes, etc (transitional: this will eventually be native format)
xmlDoc = safeParseXML(string.join(record.XmlData, r''))
if not xmlDoc:
failedHandshakes += 1
responseString = 'Internal Error: cannot parse internally generated XML record'
DebugPrint(0, responseString)
DebugPrint(
0, '***********************************************************')
return responseString
xmlDoc.normalize()
# Generate the XML
record.XmlData = safeEncodeXML(xmlDoc).splitlines(True)
# Close and clean up the document
xmlDoc.unlink()
# Currently, the recordXml is in a list format, with each item being a line of xml.
# the collector web service requires the xml to be sent as a string.
# This logic here turns the xml list into a single xml string.
usageXmlString = r''
for line in record.XmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, 'UsageXml: ' + usageXmlString)
connectionProblem = connect_utils.connectionRetries > 0 or connect_utils.connectionError
if global_state.bundle_size > 1:
# Delay the sending until we have 'bundle_size' records.
responseString, response_obj = global_state.CurrentBundle.addHandshake(
usageXmlString)
else:
# Attempt to send the record to the collector. Note that this must
# be sent currently as an update, not as a handshake.
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(),
usageXmlString)
responseString = response_obj.getMessage()
DebugPrint(1, 'Response code: ' + str(response_obj.getCode()))
DebugPrint(1, 'Response message: ' + response_obj.getMessage())
# Determine if the call was successful based on the response
# code. Currently, 0 = success
if response_obj.getCode() == 0:
DebugPrint(1, 'Response indicates success, ')
successfulHandshakes += 1
if connectionProblem or sandbox_mgmt.hasMoreOutstandingRecord:
# Reprocess failed records before attempting more new ones
sandbox_mgmt.SearchOutstandingRecord()
reprocess.Reprocess()
else:
DebugPrint(1, 'Response indicates failure, ')
failedHandshakes += 1
DebugPrint(0, responseString)
DebugPrint(0,
'***********************************************************')
return responseString
def SendXMLFiles(fileDir, removeOriginal=False, resourceType=None):
path = os.path.join(fileDir, '*')
files = glob.glob(path)
responseString = r''
for xmlFilename in files:
DebugPrint(
0, '***********************************************************')
if os.path.getsize(xmlFilename) == 0:
DebugPrint(0, 'File ' + xmlFilename + ' is zero-length: skipping')
RemoveFile(xmlFilename)
continue
DebugPrint(2, 'xmlFilename: ', xmlFilename)
if sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles():
responseString = 'Fatal Error: too many pending files'
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************')
return responseString
# Open the XML file
try:
xmlDoc = xml.dom.minidom.parse(xmlFilename)
except:
DebugPrint(
0,
'Failed to parse XML file ',
xmlFilename,
'--',
sys.exc_info(),
'--',
sys.exc_info()[0],
'++',
sys.exc_info()[1],
)
xmlDoc = None
if xmlDoc:
DebugPrint(3, 'Adding information to parsed XML')
xmlDoc.normalize()
if not XmlChecker.CheckXmlDoc(xmlDoc, True, resourceType):
xmlDoc.unlink()
DebugPrint(
0, 'No unsuppressed usage records in ' + xmlFilename +
': not sending')
bundle.suppressedCount += 1
# Cleanup old records - SPC - NERSC 08/28/07
if removeOriginal:
RemoveFile(xmlFilename)
continue
# Generate the XML
xmlData = safeEncodeXML(xmlDoc)
# Close and clean up the document
xmlDoc.unlink()
else:
# XML parsing failed: slurp the file in to xmlData and
# send as-is.
DebugPrint(1, 'Backing up and sending failed XML as is.')
try:
in_file = open(xmlFilename, 'r')
except:
DebugPrint(0, 'Unable to open xmlFilename for simple read')
continue
xmlData = in_file.readlines()
in_file.close()
# Open the back up file
# fill the back up file
dirIndex = 0
success = False
f = 0
toomanyfiles = sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles(
)
toomanystaged = sandbox_mgmt.outstandingStagedTarCount >= Config.get_MaxStagedArchives(
)
if toomanyfiles and toomanystaged:
DebugPrint(
4,
'DEBUG: Too many pending files, the record has not been backed up'
)
f = sys.stdout
else:
DebugPrint(4, 'DEBUG: Back up record to send')
while not success:
(f, dirIndex) = sandbox_mgmt.OpenNewRecordFile(dirIndex)
DebugPrint(3, 'Will save in the record in:', f.name)
DebugPrint(3, 'dirIndex=', dirIndex)
if f.name == '<stdout>':
responseString = 'Fatal Error: unable to save record prior to send attempt'
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************'
)
return responseString
else:
try:
for line in xmlData:
f.write(line)
f.flush()
if f.tell() > 0:
success = True
DebugPrint(3, 'suceeded to fill: ', f.name)
else:
DebugPrint(0, 'failed to fill: ', f.name)
if f.name != '<stdout>':
sandbox_mgmt.RemoveRecordFile(f.name)
except:
DebugPrint(
0,
'failed to fill with exception: ',
f.name,
'--',
sys.exc_info(),
'--',
sys.exc_info()[0],
'++',
sys.exc_info()[1],
)
if f.name != '<stdout>':
sandbox_mgmt.RemoveRecordFile(f.name)
DebugPrint(4, 'DEBUG: Backing up record to send: OK')
if removeOriginal and f.name != '<stdout>':
RemoveFile(xmlFilename)
DebugPrint(1, 'Saved record to ' + f.name)
# Currently, the recordXml is in a list format, with each
# item being a line of xml. The collector web service
# requires the xml to be sent as a string. This logic here
# turns the xml list into a single xml string.
usageXmlString = r''
for line in xmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, 'UsageXml: ' + usageXmlString)
if global_state.bundle_size > 1 and f.name != '<stdout>':
# Delay the sending until we have 'bundle_size' records.
(responseString,
response_obj) = global_state.CurrentBundle.addRecord(
f.name, usageXmlString)
else:
# If XMLFiles can ever be anything else than Update messages,
# then one should be able to deduce messageType from the root
# element of the XML.
messageType = 'URLEncodedUpdate'
# Attempt to send the record to the collector
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(),
usageXmlString,
messageType)
responseString = response_obj.getMessage()
DebugPrint(1, 'Response code: ' + str(response_obj.getCode()))
DebugPrint(1, 'Response message: ' + response_obj.getMessage())
# Determine if the call was successful based on the
# response code. Currently, 0 = success
if response_obj.getCode() == 0:
if f.name != '<stdout>':
DebugPrint(
1, 'Response indicates success, ' + f.name +
' will be deleted')
sandbox_mgmt.RemoveRecordFile(f.name)
else:
DebugPrint(1, 'Response indicates success')
bundle.successfulSendCount += 1
else:
bundle.failedSendCount += 1
DebugPrint(
1, 'Response indicates failure, ' + f.name +
' will not be deleted')
DebugPrint(0, responseString)
DebugPrint(0,
'***********************************************************')
return responseString
def Send(record):
try:
DebugPrint(
0, '***********************************************************')
DebugPrint(4, 'DEBUG: In Send(record)')
DebugPrint(4, 'DEBUG: Printing record to send')
record.Print()
DebugPrint(4, 'DEBUG: Printing record to send: OK')
DebugPrint(
4,
'DEBUG: File Count: ' + str(sandbox_mgmt.outstandingRecordCount))
toomanyfiles = sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles(
)
if global_state.estimatedServiceBacklog > 0:
global_state.estimatedServiceBacklog -= 1
# Assemble the record into xml
DebugPrint(4, 'DEBUG: Creating XML')
record.XmlCreate()
DebugPrint(4, 'DEBUG: Creating XML: OK')
# Parse it into nodes, etc
DebugPrint(4, 'DEBUG: parsing XML')
xmlDoc = safeParseXML(string.join(record.XmlData, r''))
DebugPrint(4, 'DEBUG: parsing XML: OK')
if not xmlDoc:
responseString = 'Internal Error: cannot parse internally generated XML record'
# We intentionally do not delete the input files.
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************')
return responseString
DebugPrint(4, 'DEBUG: Checking XML content')
if not XmlChecker.CheckXmlDoc(xmlDoc, False):
DebugPrint(4, 'DEBUG: Checking XML content: BAD')
xmlDoc.unlink()
responseString = 'OK: No unsuppressed usage records in this packet: not sending'
record.QuarantineTransientInputFiles()
bundle.suppressedCount += 1
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************')
return responseString
DebugPrint(4, 'DEBUG: Checking XML content: OK')
DebugPrint(4, 'DEBUG: Normalizing XML document')
xmlDoc.normalize()
DebugPrint(4, 'DEBUG: Normalizing XML document: OK')
# Generate the XML
DebugPrint(4, 'DEBUG: Generating data to send')
record.XmlData = safeEncodeXML(xmlDoc).splitlines(True)
DebugPrint(4, 'DEBUG: Generating data to send: OK')
# Close and clean up the document2
xmlDoc.unlink()
dirIndex = 0
success = False
f = 0
DebugPrint(4, 'DEBUG: Attempt to back up record to send')
while not success:
(f, dirIndex) = sandbox_mgmt.OpenNewRecordFile(dirIndex)
DebugPrint(3, 'Will save the record in:', f.name)
DebugPrint(3, 'dirIndex=', dirIndex)
if f.name != '<stdout>':
try:
for line in record.XmlData:
f.write(line)
f.flush()
if f.tell() > 0:
success = True
DebugPrint(1, 'Saved record to ' + f.name)
else:
DebugPrint(0, 'failed to fill: ', f.name)
if f.name != '<stdout>':
sandbox_mgmt.RemoveRecordFile(f.name)
f.close()
record.RemoveTransientInputFiles()
except:
DebugPrint(
0,
'failed to fill with exception: ',
f.name,
'--',
sys.exc_info(),
'--',
sys.exc_info()[0],
'++',
sys.exc_info()[1],
)
DebugPrint(4, 'DEBUG: Backing up record to send: OK')
else:
break
# Currently, the recordXml is in a list format, with each item being a line of xml.
# the collector web service requires the xml to be sent as a string.
# This logic here turns the xml list into a single xml string.
usageXmlString = r''
for line in record.XmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, 'UsageXml: ' + usageXmlString)
connectionProblem = connect_utils.connectionRetries > 0 or connect_utils.connectionError
if global_state.bundle_size > 1 and f.name != '<stdout>':
# Delay the sending until we have 'bundle_size' records.
(responseString,
response_obj) = global_state.CurrentBundle.addRecord(
f.name, usageXmlString)
else:
# Attempt to send the record to the collector
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(),
usageXmlString)
responseString = response_obj.getMessage()
DebugPrint(1, 'Response code: ' + str(response_obj.getCode()))
DebugPrint(1, 'Response message: ' + response_obj.getMessage())
# Determine if the call was successful based on the response
# code. Currently, 0 = success
if response_obj.getCode() == 0:
if f.name != '<stdout>':
DebugPrint(
1, 'Response indicates success, ' + f.name +
' will be deleted')
sandbox_mgmt.RemoveRecordFile(f.name)
else:
record.RemoveTransientInputFiles()
DebugPrint(1, 'Response indicates success')
bundle.successfulSendCount += 1
else:
bundle.failedSendCount += 1
if toomanyfiles:
DebugPrint(
1,
'Due to too many pending files and a connection error, the following record was not sent and has not been backed up.'
)
DebugPrint(1, 'Lost record: ' + usageXmlString)
responseString = 'Fatal Error: too many pending files'
elif f.name == '<stdout>':
DebugPrint(
0,
'Record send failed and no backup made: record lost!')
responseString += '\nFatal: failed record lost!'
match = re.search(r'^<(?:[^:]*:)?RecordIdentity.*/>$',
usageXmlString, re.MULTILINE)
if match:
DebugPrint(0, match.group(0))
responseString += ('\n', match.group(0))
match = re.search(r'^<(?:[^:]*:)?GlobalJobId.*/>$',
usageXmlString, re.MULTILINE)
if match:
DebugPrint(0, match.group(0))
responseString += ('\n', match.group(0))
responseString += '\n' + usageXmlString
else:
DebugPrint(
1, 'Response indicates failure, ' + f.name +
' will not be deleted')
DebugPrint(0, responseString)
DebugPrint(
0, '***********************************************************')
if (connectionProblem or sandbox_mgmt.hasMoreOutstandingRecord) and global_state.CurrentBundle.nItems == 0 \
and response_obj.getCode() == 0:
# Reprocess failed records before attempting more new ones
sandbox_mgmt.SearchOutstandingRecord()
reprocess.Reprocess()
return responseString
except KeyboardInterrupt:
raise
except SystemExit:
raise
except Exception, e:
DebugPrint(
0,
'ERROR: ' + str(e) + ' exception caught while processing record ')
DebugPrint(0, ' This record has been LOST')
DebugPrintTraceback()
return 'ERROR: record lost due to internal error!'
def SendHandshake(record):
global successfulHandshakes
global failedHandshakes
DebugPrint(0, "***********************************************************")
# Assemble the record into xml
record.XmlCreate()
# Parse it into nodes, etc (transitional: this will eventually be native format)
xmlDoc = safeParseXML(string.join(record.XmlData, r""))
if not xmlDoc:
failedHandshakes += 1
responseString = "Internal Error: cannot parse internally generated XML record"
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
xmlDoc.normalize()
# Generate the XML
record.XmlData = safeEncodeXML(xmlDoc).splitlines(True)
# Close and clean up the document
xmlDoc.unlink()
# Currently, the recordXml is in a list format, with each item being a line of xml.
# the collector web service requires the xml to be sent as a string.
# This logic here turns the xml list into a single xml string.
usageXmlString = r""
for line in record.XmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, "UsageXml: " + usageXmlString)
connectionProblem = connect_utils.connectionRetries > 0 or connect_utils.connectionError
if global_state.bundle_size > 1:
# Delay the sending until we have 'bundle_size' records.
responseString, response_obj = global_state.CurrentBundle.addHandshake(usageXmlString)
else:
# Attempt to send the record to the collector. Note that this must
# be sent currently as an update, not as a handshake.
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(), usageXmlString)
responseString = response_obj.getMessage()
DebugPrint(1, "Response code: " + str(response_obj.getCode()))
DebugPrint(1, "Response message: " + response_obj.getMessage())
# Determine if the call was successful based on the response
# code. Currently, 0 = success
if response_obj.getCode() == 0:
DebugPrint(1, "Response indicates success, ")
successfulHandshakes += 1
if connectionProblem or sandbox_mgmt.hasMoreOutstandingRecord:
# Reprocess failed records before attempting more new ones
sandbox_mgmt.SearchOutstandingRecord()
reprocess.Reprocess()
else:
DebugPrint(1, "Response indicates failure, ")
failedHandshakes += 1
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
def ReprocessList():
currentFailedCount = 0
currentSuccessCount = 0
currentBundledCount = 0
prevBundled = global_state.CurrentBundle.nItems
prevQuarantine = bundle.quarantinedFiles
responseString = r''
# Loop through and try to send any outstanding records
filenames = sandbox_mgmt.outstandingRecord.keys()
filenames.sort()
for failedRecord in filenames:
if connect_utils.connectionError:
# Fail record without attempting to send.
bundle.failedReprocessCount += 1
currentFailedCount += 1
continue
xmlData = None
# if os.path.isfile(failedRecord):
DebugPrint(1, 'Reprocessing: ' + failedRecord)
# Read the contents of the file into a string of xml
try:
in_file = open(failedRecord, 'r')
xmlData = in_file.read()
in_file.close()
except:
DebugPrint(1, 'Reprocess failure: unable to read file: ' + failedRecord)
responseString = responseString + '\nUnable to read from ' + failedRecord
bundle.failedReprocessCount += 1
currentFailedCount += 1
sandbox_mgmt.RemoveRecordFile(failedRecord)
del sandbox_mgmt.outstandingRecord[failedRecord]
continue
if not xmlData:
DebugPrint(1, 'Reprocess failure: ' + failedRecord + ' was empty: skip send')
responseString = responseString + '\nEmpty file ' + failedRecord + ': XML not sent'
bundle.failedReprocessCount += 1
currentFailedCount += 1
sandbox_mgmt.RemoveRecordFile(failedRecord)
del sandbox_mgmt.outstandingRecord[failedRecord]
continue
if global_state.bundle_size > 1:
_, response_obj = global_state.CurrentBundle.addReprocess(failedRecord, xmlData)
if response_obj.getCode() == response.Response.BundleNotSupported:
# The bundling was canceled, Reprocess was called recursively, we are done.
break
elif response_obj.getCode() != 0:
currentFailedCount += global_state.CurrentBundle.nLastProcessed - prevBundled
currentBundledCount = global_state.CurrentBundle.nItems
prevBundled = 0
if connect_utils.connectionError:
DebugPrint(1,
'Connection problems: reprocessing suspended; new record processing shall continue'
)
else:
if global_state.CurrentBundle.nReprocessed == 0:
currentSuccessCount += global_state.CurrentBundle.nLastProcessed - prevBundled
currentBundledCount = global_state.CurrentBundle.nItems
prevBundled = 0
else:
currentBundledCount += 1
else:
# Send the xml to the collector for processing
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(), xmlData)
# Determine if the call succeeded, and remove the file if it did
if response_obj.getCode() == 0:
DebugPrint(3, 'Processing bundle Response code for ' + failedRecord + ': '
+ str(response_obj.getCode()))
DebugPrint(3, 'Processing bundle Response message for ' + failedRecord + ': '
+ response_obj.getMessage())
DebugPrint(1, 'Response indicates success, ' + failedRecord + ' will be deleted')
currentSuccessCount += 1
bundle.successfulReprocessCount += 1
sandbox_mgmt.RemoveRecordFile(failedRecord)
del sandbox_mgmt.outstandingRecord[failedRecord]
else:
DebugPrint(1, 'Processing bundle Response code for ' + failedRecord + ': '
+ str(response_obj.getCode()))
DebugPrint(1, 'Processing bundle Response message for ' + failedRecord + ': '
+ response_obj.getMessage())
currentFailedCount += 1
if connect_utils.connectionError:
DebugPrint(1,
'Connection problems: reprocessing suspended; new record processing shall continue'
)
bundle.failedReprocessCount += 1
if currentFailedCount == 0:
responseString = 'OK'
elif currentSuccessCount != 0:
responseString = 'Warning'
else:
responseString = 'Error'
responseString += ' - Reprocessing ' + str(currentSuccessCount) + ' record(s) uploaded, ' \
+ str(currentBundledCount) + ' bundled, ' + str(currentFailedCount) + ' failed'
DebugPrint(0, 'Reprocessing response: ' + responseString)
DebugPrint(1, 'After reprocessing: ' + str(sandbox_mgmt.outstandingRecordCount) + ' in outbox '
+ str(sandbox_mgmt.outstandingStagedRecordCount) + ' in staged outbox ' + str(sandbox_mgmt.outstandingStagedTarCount)
+ ' tar files')
return (responseString, currentSuccessCount > 0 or currentBundledCount == len(sandbox_mgmt.outstandingRecord.keys())
or prevQuarantine != bundle.quarantinedFiles)
def Send(record):
try:
DebugPrint(0, "***********************************************************")
DebugPrint(4, "DEBUG: In Send(record)")
DebugPrint(4, "DEBUG: Printing record to send")
record.Print()
DebugPrint(4, "DEBUG: Printing record to send: OK")
DebugPrint(4, "DEBUG: File Count: " + str(sandbox_mgmt.outstandingRecordCount))
toomanyfiles = sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles()
if global_state.estimatedServiceBacklog > 0:
global_state.estimatedServiceBacklog -= 1
# Assemble the record into xml
DebugPrint(4, "DEBUG: Creating XML")
record.XmlCreate()
DebugPrint(4, "DEBUG: Creating XML: OK")
# Parse it into nodes, etc
DebugPrint(4, "DEBUG: parsing XML")
xmlDoc = safeParseXML(string.join(record.XmlData, r""))
DebugPrint(4, "DEBUG: parsing XML: OK")
if not xmlDoc:
responseString = "Internal Error: cannot parse internally generated XML record"
# We intentionally do not delete the input files.
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
DebugPrint(4, "DEBUG: Checking XML content")
if not XmlChecker.CheckXmlDoc(xmlDoc, False):
DebugPrint(4, "DEBUG: Checking XML content: BAD")
xmlDoc.unlink()
responseString = "OK: No unsuppressed usage records in this packet: not sending"
record.QuarantineTransientInputFiles()
bundle.suppressedCount += 1
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
DebugPrint(4, "DEBUG: Checking XML content: OK")
DebugPrint(4, "DEBUG: Normalizing XML document")
xmlDoc.normalize()
DebugPrint(4, "DEBUG: Normalizing XML document: OK")
# Generate the XML
DebugPrint(4, "DEBUG: Generating data to send")
record.XmlData = safeEncodeXML(xmlDoc).splitlines(True)
DebugPrint(4, "DEBUG: Generating data to send: OK")
# Close and clean up the document2
xmlDoc.unlink()
dirIndex = 0
success = False
f = 0
DebugPrint(4, "DEBUG: Attempt to back up record to send")
while not success:
(f, dirIndex) = sandbox_mgmt.OpenNewRecordFile(dirIndex)
DebugPrint(3, "Will save the record in:", f.name)
DebugPrint(3, "dirIndex=", dirIndex)
if f.name != "<stdout>":
try:
for line in record.XmlData:
f.write(line)
f.flush()
if f.tell() > 0:
success = True
DebugPrint(1, "Saved record to " + f.name)
else:
DebugPrint(0, "failed to fill: ", f.name)
if f.name != "<stdout>":
sandbox_mgmt.RemoveRecordFile(f.name)
f.close()
record.RemoveTransientInputFiles()
except:
DebugPrint(
0,
"failed to fill with exception: ",
f.name,
"--",
sys.exc_info(),
"--",
sys.exc_info()[0],
"++",
sys.exc_info()[1],
)
DebugPrint(4, "DEBUG: Backing up record to send: OK")
else:
break
# Currently, the recordXml is in a list format, with each item being a line of xml.
# the collector web service requires the xml to be sent as a string.
# This logic here turns the xml list into a single xml string.
usageXmlString = r""
for line in record.XmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, "UsageXml: " + usageXmlString)
connectionProblem = connect_utils.connectionRetries > 0 or connect_utils.connectionError
if global_state.bundle_size > 1 and f.name != "<stdout>":
# Delay the sending until we have 'bundle_size' records.
(responseString, response_obj) = global_state.CurrentBundle.addRecord(f.name, usageXmlString)
else:
# Attempt to send the record to the collector
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(), usageXmlString)
responseString = response_obj.getMessage()
DebugPrint(1, "Response code: " + str(response_obj.getCode()))
DebugPrint(1, "Response message: " + response_obj.getMessage())
# Determine if the call was successful based on the response
# code. Currently, 0 = success
if response_obj.getCode() == 0:
if f.name != "<stdout>":
DebugPrint(1, "Response indicates success, " + f.name + " will be deleted")
sandbox_mgmt.RemoveRecordFile(f.name)
else:
record.RemoveTransientInputFiles()
DebugPrint(1, "Response indicates success")
bundle.successfulSendCount += 1
else:
bundle.failedSendCount += 1
if toomanyfiles:
DebugPrint(
1,
"Due to too many pending files and a connection error, the following record was not sent and has not been backed up.",
)
DebugPrint(1, "Lost record: " + usageXmlString)
responseString = "Fatal Error: too many pending files"
elif f.name == "<stdout>":
DebugPrint(0, "Record send failed and no backup made: record lost!")
responseString += "\nFatal: failed record lost!"
match = re.search(r"^<(?:[^:]*:)?RecordIdentity.*/>$", usageXmlString, re.MULTILINE)
if match:
DebugPrint(0, match.group(0))
responseString += ("\n", match.group(0))
match = re.search(r"^<(?:[^:]*:)?GlobalJobId.*/>$", usageXmlString, re.MULTILINE)
if match:
DebugPrint(0, match.group(0))
responseString += ("\n", match.group(0))
responseString += "\n" + usageXmlString
else:
DebugPrint(1, "Response indicates failure, " + f.name + " will not be deleted")
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
if (
(connectionProblem or sandbox_mgmt.hasMoreOutstandingRecord)
and global_state.CurrentBundle.nItems == 0
and response_obj.getCode() == 0
):
# Reprocess failed records before attempting more new ones
sandbox_mgmt.SearchOutstandingRecord()
reprocess.Reprocess()
return responseString
except KeyboardInterrupt:
raise
except SystemExit:
raise
except Exception, e:
DebugPrint(0, "ERROR: " + str(e) + " exception caught while processing record ")
DebugPrint(0, " This record has been LOST")
DebugPrintTraceback()
return "ERROR: record lost due to internal error!"
def SendXMLFiles(fileDir, removeOriginal=False, resourceType=None):
path = os.path.join(fileDir, "*")
files = glob.glob(path)
responseString = r""
for xmlFilename in files:
DebugPrint(0, "***********************************************************")
if os.path.getsize(xmlFilename) == 0:
DebugPrint(0, "File " + xmlFilename + " is zero-length: skipping")
RemoveFile(xmlFilename)
continue
DebugPrint(2, "xmlFilename: ", xmlFilename)
if sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles():
responseString = "Fatal Error: too many pending files"
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
# Open the XML file
try:
xmlDoc = xml.dom.minidom.parse(xmlFilename)
except:
DebugPrint(
0,
"Failed to parse XML file ",
xmlFilename,
"--",
sys.exc_info(),
"--",
sys.exc_info()[0],
"++",
sys.exc_info()[1],
)
xmlDoc = None
if xmlDoc:
DebugPrint(3, "Adding information to parsed XML")
xmlDoc.normalize()
if not XmlChecker.CheckXmlDoc(xmlDoc, True, resourceType):
xmlDoc.unlink()
DebugPrint(0, "No unsuppressed usage records in " + xmlFilename + ": not sending")
bundle.suppressedCount += 1
# Cleanup old records - SPC - NERSC 08/28/07
if removeOriginal:
RemoveFile(xmlFilename)
continue
# Generate the XML
xmlData = safeEncodeXML(xmlDoc)
# Close and clean up the document
xmlDoc.unlink()
else:
# XML parsing failed: slurp the file in to xmlData and
# send as-is.
DebugPrint(1, "Backing up and sending failed XML as is.")
try:
in_file = open(xmlFilename, "r")
except:
DebugPrint(0, "Unable to open xmlFilename for simple read")
continue
xmlData = in_file.readlines()
in_file.close()
# Open the back up file
# fill the back up file
dirIndex = 0
success = False
f = 0
toomanyfiles = sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles()
toomanystaged = sandbox_mgmt.outstandingStagedTarCount >= Config.get_MaxStagedArchives()
if toomanyfiles and toomanystaged:
DebugPrint(4, "DEBUG: Too many pending files, the record has not been backed up")
f = sys.stdout
else:
DebugPrint(4, "DEBUG: Back up record to send")
while not success:
(f, dirIndex) = sandbox_mgmt.OpenNewRecordFile(dirIndex)
DebugPrint(3, "Will save in the record in:", f.name)
DebugPrint(3, "dirIndex=", dirIndex)
if f.name == "<stdout>":
responseString = "Fatal Error: unable to save record prior to send attempt"
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
else:
try:
for line in xmlData:
f.write(line)
f.flush()
if f.tell() > 0:
success = True
DebugPrint(3, "suceeded to fill: ", f.name)
else:
DebugPrint(0, "failed to fill: ", f.name)
if f.name != "<stdout>":
sandbox_mgmt.RemoveRecordFile(f.name)
except:
DebugPrint(
0,
"failed to fill with exception: ",
f.name,
"--",
sys.exc_info(),
"--",
sys.exc_info()[0],
"++",
sys.exc_info()[1],
)
if f.name != "<stdout>":
sandbox_mgmt.RemoveRecordFile(f.name)
DebugPrint(4, "DEBUG: Backing up record to send: OK")
if removeOriginal and f.name != "<stdout>":
RemoveFile(xmlFilename)
DebugPrint(1, "Saved record to " + f.name)
# Currently, the recordXml is in a list format, with each
# item being a line of xml. The collector web service
# requires the xml to be sent as a string. This logic here
# turns the xml list into a single xml string.
usageXmlString = r""
for line in xmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, "UsageXml: " + usageXmlString)
if global_state.bundle_size > 1 and f.name != "<stdout>":
# Delay the sending until we have 'bundle_size' records.
(responseString, response_obj) = global_state.CurrentBundle.addRecord(f.name, usageXmlString)
else:
# If XMLFiles can ever be anything else than Update messages,
# then one should be able to deduce messageType from the root
# element of the XML.
messageType = "URLEncodedUpdate"
# Attempt to send the record to the collector
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(), usageXmlString, messageType)
responseString = response_obj.getMessage()
DebugPrint(1, "Response code: " + str(response_obj.getCode()))
DebugPrint(1, "Response message: " + response_obj.getMessage())
# Determine if the call was successful based on the
# response code. Currently, 0 = success
if response_obj.getCode() == 0:
if f.name != "<stdout>":
DebugPrint(1, "Response indicates success, " + f.name + " will be deleted")
sandbox_mgmt.RemoveRecordFile(f.name)
else:
DebugPrint(1, "Response indicates success")
bundle.successfulSendCount += 1
else:
bundle.failedSendCount += 1
DebugPrint(1, "Response indicates failure, " + f.name + " will not be deleted")
DebugPrint(0, responseString)
DebugPrint(0, "***********************************************************")
return responseString
def ProcessBundle(bundle):
global successfulHandshakes
global successfulSendCount
global failedHandshakes
global failedSendCount
global successfulReprocessCount
global successfulBundleCount
global failedReprocessCount
global quarantinedFiles
global failedBundleCount
responseString = r''
# Loop through and try to send any outstanding records
bundleData = '''<?xml version="1.0" encoding="UTF-8"?>
<RecordEnvelope>
'''
for item in bundle.content:
xmlData = None
filename = item[0]
xmlData = item[1]
DebugPrint(1, 'Processing bundle file: ' + filename)
if xmlData == r'':
# Read the contents of the file into a string of xml
try:
in_file = open(filename, 'r')
xmlData = in_file.read()
in_file.close()
except:
DebugPrint(
1, 'Processing bundle failure: unable to read file: ' +
filename)
responseString = responseString + '\nUnable to read from ' + filename
failedBundleCount += 1
continue
if not xmlData:
DebugPrint(
1, 'Processing bundle failure: ' + filename +
' was empty: skip send')
responseString = responseString + '\nEmpty file ' + filename + ': XML not sent'
failedBundleCount += 1
continue
xmlData = __xmlintroRemove.sub(r'', xmlData)
bundleData = bundleData + xmlData + '\n'
# if (len(bundleData)==0):
# bundleData = xmlData
# else:
# bundleData = bundleData + '|' + xmlData
bundleData = bundleData + '</RecordEnvelope>'
# Send the xml to the collector for processing
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(),
bundleData, 'multiupdate')
DebugPrint(
2, 'Processing bundle Response code: ' + str(response_obj.getCode()))
DebugPrint(
2, 'Processing bundle Response message: ' + response_obj.getMessage())
if response_obj.getCode() == response.Response.BundleNotSupported:
DebugPrint(
0,
"Collector is too old to handle 'bundles', reverting to sending individual records."
)
global_state.bundle_size = 0
bundle.nLastProcessed = 0
hasHandshake = bundle.nHandshakes > 0
bundle.clear()
if hasHandshake:
# Done to break circular dependency between send and bundle
__import__("gratia.common.send").common.send.Handshake()
else:
sandbox_mgmt.SearchOutstandingRecord()
# Done to break circular dependency between bundle and reprocess
__import__("gratia.common.reprocess").common.reprocess.Reprocess()
return 'Bundling has been canceled.', response_obj
elif response_obj.getCode() == response.Response.PostTooLarge:
if bundle.nItems > 1:
# We let a large record to be added to already too many data.
# Let's try to restrict more the size of the record
Bundle.decreaseMaxPostSize(0.9)
#__maxPostSize = 0.9 * Bundle.__maxPostSize
elif bundle.nItems == 1:
DebugPrint(
0,
'Error: a record is larger than the Collector can receive. (' +
str(len(bundleData) * 10 / 1000 / 1000 / 10.0) +
'Mb vs 2Mb). Record will be Quarantined.')
quarantinedFiles += 1
sandbox_mgmt.QuarantineFile(bundle.content[0][0], False)
else:
DebugPrint(
0,
"Internal error, got a 'too large of a post' response eventhough we have no record at all!"
)
responseString = 'Processed bundle with ' + str(
bundle.nItems) + ' records: ' + response_obj.getMessage()
# Determine if the call succeeded, and remove the file if it did
if response_obj.getCode() == 0:
successfulSendCount += bundle.nRecords
successfulHandshakes += bundle.nHandshakes
successfulReprocessCount += bundle.nReprocessed
successfulBundleCount += 1
for item in bundle.content:
filename = item[0]
if filename != r'':
DebugPrint(
1, 'Bundle response indicates success, ' + filename +
' will be deleted')
sandbox_mgmt.RemoveRecordFile(filename)
responseString = 'OK - ' + responseString
else:
DebugPrint(
1,
'Response indicates failure, the following files will not be deleted:'
)
for item in bundle.content:
filename = item[0]
if filename != r'':
DebugPrint(1, ' ' + filename)
failedSendCount += bundle.nRecords
failedHandshakes += bundle.nHandshakes
failedReprocessCount += bundle.nReprocessed
failedBundleCount += 1
bundle.nLastProcessed = bundle.nItems
bundle.clear()
return responseString, response_obj
def ReprocessList():
currentFailedCount = 0
currentSuccessCount = 0
currentBundledCount = 0
prevBundled = global_state.CurrentBundle.nItems
prevQuarantine = bundle.quarantinedFiles
responseString = r''
# Loop through and try to send any outstanding records
filenames = list(sandbox_mgmt.outstandingRecord.keys())
filenames.sort()
for failedRecord in filenames:
if connect_utils.connectionError:
# Fail record without attempting to send.
bundle.failedReprocessCount += 1
currentFailedCount += 1
continue
xmlData = None
# if os.path.isfile(failedRecord):
DebugPrint(1, 'Reprocessing: ' + failedRecord)
# Read the contents of the file into a string of xml
try:
in_file = open(failedRecord, 'r')
xmlData = in_file.read()
in_file.close()
except:
DebugPrint(
1, 'Reprocess failure: unable to read file: ' + failedRecord)
responseString = responseString + '\nUnable to read from ' + failedRecord
bundle.failedReprocessCount += 1
currentFailedCount += 1
sandbox_mgmt.RemoveRecordFile(failedRecord)
del sandbox_mgmt.outstandingRecord[failedRecord]
continue
if not xmlData:
DebugPrint(
1,
'Reprocess failure: ' + failedRecord + ' was empty: skip send')
responseString = responseString + '\nEmpty file ' + failedRecord + ': XML not sent'
bundle.failedReprocessCount += 1
currentFailedCount += 1
sandbox_mgmt.RemoveRecordFile(failedRecord)
del sandbox_mgmt.outstandingRecord[failedRecord]
continue
if global_state.bundle_size > 1:
_, response_obj = global_state.CurrentBundle.addReprocess(
failedRecord, xmlData)
if response_obj.getCode() == response.Response.BundleNotSupported:
# The bundling was canceled, Reprocess was called recursively, we are done.
break
elif response_obj.getCode() != 0:
currentFailedCount += global_state.CurrentBundle.nLastProcessed - prevBundled
currentBundledCount = global_state.CurrentBundle.nItems
prevBundled = 0
if connect_utils.connectionError:
DebugPrint(
1,
'Connection problems: reprocessing suspended; new record processing shall continue'
)
else:
if global_state.CurrentBundle.nReprocessed == 0:
currentSuccessCount += global_state.CurrentBundle.nLastProcessed - prevBundled
currentBundledCount = global_state.CurrentBundle.nItems
prevBundled = 0
else:
currentBundledCount += 1
else:
# Send the xml to the collector for processing
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(),
xmlData)
# Determine if the call succeeded, and remove the file if it did
if response_obj.getCode() == 0:
DebugPrint(
3, 'Processing bundle Response code for ' + failedRecord +
': ' + str(response_obj.getCode()))
DebugPrint(
3, 'Processing bundle Response message for ' +
failedRecord + ': ' + response_obj.getMessage())
DebugPrint(
1, 'Response indicates success, ' + failedRecord +
' will be deleted')
currentSuccessCount += 1
bundle.successfulReprocessCount += 1
sandbox_mgmt.RemoveRecordFile(failedRecord)
del sandbox_mgmt.outstandingRecord[failedRecord]
else:
DebugPrint(
1, 'Processing bundle Response code for ' + failedRecord +
': ' + str(response_obj.getCode()))
DebugPrint(
1, 'Processing bundle Response message for ' +
failedRecord + ': ' + response_obj.getMessage())
currentFailedCount += 1
if connect_utils.connectionError:
DebugPrint(
1,
'Connection problems: reprocessing suspended; new record processing shall continue'
)
bundle.failedReprocessCount += 1
if currentFailedCount == 0:
responseString = 'OK'
elif currentSuccessCount != 0:
responseString = 'Warning'
else:
responseString = 'Error'
responseString += ' - Reprocessing ' + str(currentSuccessCount) + ' record(s) uploaded, ' \
+ str(currentBundledCount) + ' bundled, ' + str(currentFailedCount) + ' failed'
DebugPrint(0, 'Reprocessing response: ' + responseString)
DebugPrint(
1, 'After reprocessing: ' + str(sandbox_mgmt.outstandingRecordCount) +
' in outbox ' + str(sandbox_mgmt.outstandingStagedRecordCount) +
' in staged outbox ' + str(sandbox_mgmt.outstandingStagedTarCount) +
' tar files')
return (responseString, currentSuccessCount > 0
or currentBundledCount == len(
list(sandbox_mgmt.outstandingRecord.keys()))
or prevQuarantine != bundle.quarantinedFiles)
def ProcessBundle(bundle):
global successfulHandshakes
global successfulSendCount
global failedHandshakes
global failedSendCount
global successfulReprocessCount
global successfulBundleCount
global failedReprocessCount
global quarantinedFiles
global failedBundleCount
responseString = r''
# Loop through and try to send any outstanding records
bundleData = '''<?xml version="1.0" encoding="UTF-8"?>
<RecordEnvelope>
'''
for item in bundle.content:
xmlData = None
filename = item[0]
xmlData = item[1]
DebugPrint(1, 'Processing bundle file: ' + filename)
if xmlData == r'':
# Read the contents of the file into a string of xml
try:
in_file = open(filename, 'r')
xmlData = in_file.read()
in_file.close()
except:
DebugPrint(1, 'Processing bundle failure: unable to read file: ' + filename)
responseString = responseString + '\nUnable to read from ' + filename
failedBundleCount += 1
continue
if not xmlData:
DebugPrint(1, 'Processing bundle failure: ' + filename + ' was empty: skip send')
responseString = responseString + '\nEmpty file ' + filename + ': XML not sent'
failedBundleCount += 1
continue
xmlData = __xmlintroRemove.sub(r'', xmlData)
bundleData = bundleData + xmlData + '\n'
# if (len(bundleData)==0):
# bundleData = xmlData
# else:
# bundleData = bundleData + '|' + xmlData
bundleData = bundleData + '</RecordEnvelope>'
# Send the xml to the collector for processing
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(), bundleData, 'multiupdate')
DebugPrint(2, 'Processing bundle Response code: ' + str(response_obj.getCode()))
DebugPrint(2, 'Processing bundle Response message: ' + response_obj.getMessage())
if response_obj.getCode() == response.Response.BundleNotSupported:
DebugPrint(0, "Collector is too old to handle 'bundles', reverting to sending individual records.")
global_state.bundle_size = 0
bundle.nLastProcessed = 0
hasHandshake = bundle.nHandshakes > 0
bundle.clear()
if hasHandshake:
# Done to break circular dependency between send and bundle
__import__("gratia.common.send").common.send.Handshake()
else:
sandbox_mgmt.SearchOutstandingRecord()
# Done to break circular dependency between bundle and reprocess
__import__("gratia.common.reprocess").common.reprocess.Reprocess()
return 'Bundling has been canceled.', response_obj
elif response_obj.getCode() == response.Response.PostTooLarge:
if bundle.nItems > 1:
# We let a large record to be added to already too many data.
# Let's try to restrict more the size of the record
Bundle.decreaseMaxPostSize(0.9)
#__maxPostSize = 0.9 * Bundle.__maxPostSize
elif bundle.nItems == 1:
DebugPrint(0, 'Error: a record is larger than the Collector can receive. (' + str(len(bundleData)
* 10 / 1000 / 1000 / 10.0) + 'Mb vs 2Mb). Record will be Quarantined.')
quarantinedFiles += 1
sandbox_mgmt.QuarantineFile(bundle.content[0][0], False)
else:
DebugPrint(0,
"Internal error, got a 'too large of a post' response eventhough we have no record at all!"
)
responseString = 'Processed bundle with ' + str(bundle.nItems) + ' records: ' + response_obj.getMessage()
# Determine if the call succeeded, and remove the file if it did
if response_obj.getCode() == 0:
successfulSendCount += bundle.nRecords
successfulHandshakes += bundle.nHandshakes
successfulReprocessCount += bundle.nReprocessed
successfulBundleCount += 1
for item in bundle.content:
filename = item[0]
if filename != r'':
DebugPrint(1, 'Bundle response indicates success, ' + filename + ' will be deleted')
sandbox_mgmt.RemoveRecordFile(filename)
responseString = 'OK - ' + responseString
else:
DebugPrint(1, 'Response indicates failure, the following files will not be deleted:')
for item in bundle.content:
filename = item[0]
if filename != r'':
DebugPrint(1, ' ' + filename)
failedSendCount += bundle.nRecords
failedHandshakes += bundle.nHandshakes
failedReprocessCount += bundle.nReprocessed
failedBundleCount += 1
bundle.nLastProcessed = bundle.nItems
bundle.clear()
return responseString, response_obj