def RemoveTransientInputFiles(self):
''' Delete all the transient input files. '''
for filename in self.TransientInputFiles:
DebugPrint(1, 'Deleting transient input file: ' + filename)
RemoveFile(filename)
self.TransientInputFiles = []
def QuarantineTransientInputFiles(self):
''' Copy to a quarantine directories any of the input files '''
quarantinedir = os.path.join(Config.get_DataFolder(), "quarantine")
Mkdir(quarantinedir)
for filename in self.TransientInputFiles:
DebugPrint(
1, 'Moving transient input file: ' + filename +
' to quarantine in ' + quarantinedir)
shutil.copy2(filename, quarantinedir)
RemoveFile(filename)
self.TransientInputFiles = []
def RemoveRecordFile(filename):
# Remove a record file and reduce the oustanding record count
global outstandingRecordCount
global outstandingStagedRecordCount
if RemoveFile(filename):
# Decrease the count only if the file was really removed
dirname = os.path.dirname(filename)
if os.path.basename(dirname) == 'outbox' and os.path.basename(os.path.dirname(dirname)) == 'staged':
DebugPrint(3, 'Remove the staged record: ' + filename)
outstandingStagedRecordCount += -1
else:
outstandingRecordCount += -1
DebugPrint(3, 'Remove the record: ' + filename)
def SendXMLFiles(fileDir, removeOriginal=False, resourceType=None):
path = os.path.join(fileDir, '*')
files = glob.glob(path)
responseString = r''
for xmlFilename in files:
DebugPrint(
0, '***********************************************************')
if os.path.getsize(xmlFilename) == 0:
DebugPrint(0, 'File ' + xmlFilename + ' is zero-length: skipping')
RemoveFile(xmlFilename)
continue
DebugPrint(2, 'xmlFilename: ', xmlFilename)
if sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles():
responseString = 'Fatal Error: too many pending files'
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************')
return responseString
# Open the XML file
try:
xmlDoc = xml.dom.minidom.parse(xmlFilename)
except:
DebugPrint(
0,
'Failed to parse XML file ',
xmlFilename,
'--',
sys.exc_info(),
'--',
sys.exc_info()[0],
'++',
sys.exc_info()[1],
)
xmlDoc = None
if xmlDoc:
DebugPrint(3, 'Adding information to parsed XML')
xmlDoc.normalize()
if not XmlChecker.CheckXmlDoc(xmlDoc, True, resourceType):
xmlDoc.unlink()
DebugPrint(
0, 'No unsuppressed usage records in ' + xmlFilename +
': not sending')
bundle.suppressedCount += 1
# Cleanup old records - SPC - NERSC 08/28/07
if removeOriginal:
RemoveFile(xmlFilename)
continue
# Generate the XML
xmlData = safeEncodeXML(xmlDoc)
# Close and clean up the document
xmlDoc.unlink()
else:
# XML parsing failed: slurp the file in to xmlData and
# send as-is.
DebugPrint(1, 'Backing up and sending failed XML as is.')
try:
in_file = open(xmlFilename, 'r')
except:
DebugPrint(0, 'Unable to open xmlFilename for simple read')
continue
xmlData = in_file.readlines()
in_file.close()
# Open the back up file
# fill the back up file
dirIndex = 0
success = False
f = 0
toomanyfiles = sandbox_mgmt.outstandingRecordCount >= Config.get_MaxPendingFiles(
)
toomanystaged = sandbox_mgmt.outstandingStagedTarCount >= Config.get_MaxStagedArchives(
)
if toomanyfiles and toomanystaged:
DebugPrint(
4,
'DEBUG: Too many pending files, the record has not been backed up'
)
f = sys.stdout
else:
DebugPrint(4, 'DEBUG: Back up record to send')
while not success:
(f, dirIndex) = sandbox_mgmt.OpenNewRecordFile(dirIndex)
DebugPrint(3, 'Will save in the record in:', f.name)
DebugPrint(3, 'dirIndex=', dirIndex)
if f.name == '<stdout>':
responseString = 'Fatal Error: unable to save record prior to send attempt'
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************'
)
return responseString
else:
try:
for line in xmlData:
f.write(line)
f.flush()
if f.tell() > 0:
success = True
DebugPrint(3, 'suceeded to fill: ', f.name)
else:
DebugPrint(0, 'failed to fill: ', f.name)
if f.name != '<stdout>':
sandbox_mgmt.RemoveRecordFile(f.name)
except:
DebugPrint(
0,
'failed to fill with exception: ',
f.name,
'--',
sys.exc_info(),
'--',
sys.exc_info()[0],
'++',
sys.exc_info()[1],
)
if f.name != '<stdout>':
sandbox_mgmt.RemoveRecordFile(f.name)
DebugPrint(4, 'DEBUG: Backing up record to send: OK')
if removeOriginal and f.name != '<stdout>':
RemoveFile(xmlFilename)
DebugPrint(1, 'Saved record to ' + f.name)
# Currently, the recordXml is in a list format, with each
# item being a line of xml. The collector web service
# requires the xml to be sent as a string. This logic here
# turns the xml list into a single xml string.
usageXmlString = r''
for line in xmlData:
usageXmlString = usageXmlString + line
DebugPrint(3, 'UsageXml: ' + usageXmlString)
if global_state.bundle_size > 1 and f.name != '<stdout>':
# Delay the sending until we have 'bundle_size' records.
(responseString,
response_obj) = global_state.CurrentBundle.addRecord(
f.name, usageXmlString)
else:
# If XMLFiles can ever be anything else than Update messages,
# then one should be able to deduce messageType from the root
# element of the XML.
messageType = 'URLEncodedUpdate'
# Attempt to send the record to the collector
response_obj = connect_utils.sendUsageXML(Config.get_ProbeName(),
usageXmlString,
messageType)
responseString = response_obj.getMessage()
DebugPrint(1, 'Response code: ' + str(response_obj.getCode()))
DebugPrint(1, 'Response message: ' + response_obj.getMessage())
# Determine if the call was successful based on the
# response code. Currently, 0 = success
if response_obj.getCode() == 0:
if f.name != '<stdout>':
DebugPrint(
1, 'Response indicates success, ' + f.name +
' will be deleted')
sandbox_mgmt.RemoveRecordFile(f.name)
else:
DebugPrint(1, 'Response indicates success')
bundle.successfulSendCount += 1
else:
bundle.failedSendCount += 1
DebugPrint(
1, 'Response indicates failure, ' + f.name +
' will not be deleted')
DebugPrint(0, responseString)
DebugPrint(0,
'***********************************************************')
return responseString
def RemoveOldFiles(nDays=31, globexp=None, req_maxsize=0):
if not globexp:
return
# Get the list of all files in the log directory
files = glob.glob(globexp)
if not files:
return
DebugPrint(3, ' Will check the files: ', files)
cutoff = time.time() - nDays * 24 * 3600
totalsize = 0
date_file_list = []
for oldfile in files:
if not os.path.isfile(oldfile):
continue
lastmod_date = os.path.getmtime(oldfile)
if lastmod_date < cutoff:
DebugPrint(2, 'Will remove: ' + oldfile)
RemoveFile(oldfile)
else:
size = os.path.getsize(oldfile)
totalsize += size
date_file_tuple = (lastmod_date, size, oldfile)
date_file_list.append(date_file_tuple)
if len(date_file_list) == 0:
# No more files.
return
dirname = os.path.dirname(date_file_list[0][2])
statfs = os.statvfs(dirname)
disksize = statfs.f_blocks
freespace = statfs.f_bfree
ourblocks = totalsize / statfs.f_frsize
percent = ourblocks * 100.0 / disksize
if percent < 1:
DebugPrint(
1, dirname + ' uses ' + niceNum(percent, 1e-3) +
'% and there is ' + niceNum(freespace * 100 / disksize) + '% free')
else:
DebugPrint(
1, dirname + ' uses ' + niceNum(percent, 0.10000000000000001) +
'% and there is ' + niceNum(freespace * 100 / disksize) + '% free')
minfree = 0.10000000000000001 * disksize # We want the disk to be no fuller than 95%
# We want the directory to not be artificially reduced below 5% because other things are filling up the disk.
minuse = 0.05 * disksize
calc_maxsize = req_maxsize
if freespace < minfree:
# The disk is quite full
if ourblocks > minuse:
# We already use more than 5%, let's see how much we can delete to get under 95% full but not under 5% of
# our own use
target = minfree - freespace # We would like to remove than much
if ourblocks - target < minuse:
# But it would take us under 5%, so do what we can
calc_maxsize = minuse
else:
calc_maxsize = ourblocks - target
if 0 < req_maxsize and req_maxsize < calc_maxsize * statfs.f_frsize:
calc_maxsize = req_maxsize
else:
DebugPrint(
4,
"DEBUG: The disk is quite full and this directory is 'large' attempting to reduce from "
+ niceNum(totalsize / 1000000) + 'Mb to ' +
niceNum(calc_maxsize / 1000000) + 'Mb.')
calc_maxsize = calc_maxsize * statfs.f_frsize
if calc_maxsize > 0 and totalsize > calc_maxsize:
DebugPrint(
1, 'Cleaning up directory due to space overflow: ' +
niceNum(totalsize / 1e6, 0.10000000000000001),
'Mb for a limit of ',
niceNum(calc_maxsize / 1e6, 0.10000000000000001), ' Mb.')
calc_maxsize = 0.8 * calc_maxsize
date_file_list.sort()
# To get the newest first (for debugging purpose only)
# date_file_list.reverse()
currentLogFile = LogFileName()
for file_tuple in date_file_list:
DebugPrint(2, 'Will remove: ' + file_tuple[2])
RemoveFile(file_tuple[2])
totalsize = totalsize - file_tuple[1]
if currentLogFile == file_tuple[2]:
# We delete the current log file! Let's record this explicitly!
DebugPrint(0,
'EMERGENCY DELETION AND TRUNCATION OF LOG FILES.')
DebugPrint(
0, 'Current log file was too large: ' +
niceNum(file_tuple[1] / 1000000) + 'Mb.')
DebugPrint(0, 'All prior information has been lost.')
if totalsize < calc_maxsize:
return
def SearchOutstandingRecord():
'''Search the list of backup directories for'''
global hasMoreOutstandingRecord
global outstandingRecordCount
global outstandingStagedTarCount
global outstandingStagedRecordCount
outstandingRecord.clear()
outstandingRecordCount = 0
outstandingStagedTarCount = 0
outstandingStagedRecordCount = 0
fragment = Config.getFilenameFragment()
DebugPrint(4, 'DEBUG: Starting SearchOutstandingRecord')
for current_dir in backupDirList:
DebugPrint(4, 'DEBUG: SearchOutstandingRecord ' + current_dir)
DebugPrint(
4, 'DEBUG: Middle of SearchOutstandingRecord outbox:' +
str(outstandingRecordCount) + ' staged outbox:' +
str(outstandingStagedRecordCount) + ' tarfiles:' +
str(outstandingStagedTarCount))
gratiapath = os.path.join(current_dir, 'gratiafiles')
subpath = os.path.join(gratiapath, 'subdir.' + fragment)
outbox = os.path.join(subpath, 'outbox')
staged = os.path.join(subpath, 'staged')
stagedoutbox = os.path.join(subpath, 'staged', 'outbox')
# For backward compatibility still look for the records in the top level
# gratiafiles directories.
path = os.path.join(gratiapath, 'r*.' + Config.get_GratiaExtension())
files = glob.glob(path) + glob.glob(path + '__*')
DebugPrint(4, 'DEBUG: Search add ' + str(len(files)) + ' for ' + path)
outstandingRecordCount += len(files)
for f in files:
# Legacy reprocess files or ones with the correct fragment
if re.search(
r'/?r(?:[0-9]+)?\.?[0-9]+(?:\.' + fragment + r')?\.' +
Config.get_GratiaExtension() + r'(?:__.{10})?$', f):
AddOutstandingRecord(f)
if len(outstandingRecord) >= __maxFilesToReprocess__:
break
# Record the number of tar file already on disk.
stagedfiles = glob.glob(os.path.join(staged, 'store', 'tz.*'))
outstandingStagedTarCount += len(stagedfiles)
if len(outstandingRecord) >= __maxFilesToReprocess__:
break
# Now look for the record in the probe specific subdirectory.
if ListOutstandingRecord(outbox, False):
break
prevOutstandingStagedRecordCount = outstandingStagedRecordCount
if ListOutstandingRecord(stagedoutbox, True):
break
# If total number of outstanding files is less than the number of files already in the bundle,
# Let's decompress one of the tar file (if any)
needmorefiles = outstandingStagedRecordCount == 0 or \
outstandingRecordCount + outstandingStagedRecordCount <= global_state.CurrentBundle.nFiles
if needmorefiles and len(stagedfiles) > 0:
# the staged/outbox is low on files and we have some staged tar files
in_stagedoutbox = outstandingStagedRecordCount - prevOutstandingStagedRecordCount
if in_stagedoutbox != 0 and global_state.CurrentBundle.nFiles > 0:
# This staged outbox is not empty, so let's first empty it.
# NOTE: import statement is here to break circular dependency between bundle and sandbox_mgmt
responseString, _ = __import__(
"gratia.common.bundle").common.bundle.ProcessBundle(
global_state.CurrentBundle)
DebugPrint(0, responseString)
DebugPrint(
0,
'***********************************************************'
)
if global_state.CurrentBundle.nItems > 0:
# The upload did not work, there is no need to proceed with the record collection
break
# The staged outbox is empty, we can safely untar the file without risking over-writing
# a files.
stagedfile = stagedfiles[0]
if UncompressOutbox(stagedfile, stagedoutbox):
RemoveFile(stagedfile)
else:
Mkdir(os.path.join(staged, 'quarantine'))
os.rename(
stagedfile,
os.path.join(staged, 'quarantine',
os.path.basename(stagedfile)))
outstandingStagedTarCount += -1
outstandingStagedRecordCount = prevOutstandingStagedRecordCount
if ListOutstandingRecord(stagedoutbox, True):
break
# Mark that we probably have more outstanding record to look at.
hasMoreOutstandingRecord = outstandingStagedTarCount > 0 or len(
outstandingRecord) >= __maxFilesToReprocess__
DebugPrint(4, 'DEBUG: List of Outstanding records: ',
outstandingRecord.keys())
DebugPrint(
4, 'DEBUG: After SearchOutstandingRecord outbox:' +
str(outstandingRecordCount) + ' staged outbox:' +
str(outstandingStagedRecordCount) + ' tarfiles:' +
str(outstandingStagedTarCount))