def checkFinishedStagingFiles(self):
finishedFiles = []
for file in self.__eventRanges:
try:
if len(self.__eventRanges[file].keys()) == 0:
filepath = os.path.join(self.__workDir, file)
handle = open(filepath + ".staged.reported", 'w')
finishedEventRanges = []
for chunk in pUtil.chunks(self.__eventRanges_staged[file], 100):
try:
eventRanges = []
for outputEvents in chunk:
jobId, eventRangeID, status, output = outputEvents
if eventRangeID not in finishedEventRanges:
finishedEventRanges.append(eventRangeID)
if status == 'finished':
eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": status})
if status.startswith("ERR"):
eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'failed'})
update_status, update_output = self.updateEventRanges(eventRanges)
logging.info("update Event Range: status: %s, output: %s" % (update_status, update_output))
if update_status:
update_status, update_output = self.updateEventRanges(eventRanges)
logging.info("update Event retry Range: status: %s, output: %s" % (update_status, update_output))
if update_status == 0:
try:
ret_outputs = json.loads(json.loads(update_output))
if len(ret_outputs) == len(chunk):
for i in range(len(ret_outputs)):
try:
if ret_outputs[i]:
jobId, eventRangeID, status, output = chunk[i]
logging.info("Remove %s" % output)
os.remove(output)
handle.write('{0} {1} {2} {3}\n'.format(jobId, eventRangeID, status, output))
except:
logging.warning("Failed to remove %s: %s" % (output, traceback.format_exc()))
except:
logging.warning(traceback.format_exc())
except:
logging.warning(traceback.format_exc())
handle.close()
os.rename(filepath + ".staging", filepath + ".BAK")
finishedFiles.append(file)
except:
logging.warning(traceback.format_exc())
for file in finishedFiles:
del self.__eventRanges[file]
del self.__eventRanges_staged[file]
del self.__eventRanges_faileStaged[file]
def processFile(self, outputFile):
handle = open(outputFile)
toMvFiles = []
yodaStagedFiles = []
yodaReportedFiles = []
for line in handle:
line = line.replace(" ", " ")
jobId, eventRange, status, output = line.split(" ")
if status == 'stagedOut' or status.startswith("ERR"):
yodaStagedFiles.append(line)
elif status == 'reported':
yodaReportedFiles.append(line)
yodaStagedFiles.append(line)
else:
toMvFiles.append(line)
handle.close()
if toMvFiles:
handlemv = open(outputFile + ".mv", "w")
for i in range(3):
tmpToMvFiles = toMvFiles
toMvFiles = []
for line in tmpToMvFiles:
line = line.replace(" ", " ")
jobId, eventRange, status, output = line.split(" ")
outputPath = output.split(",")[0]
newOutputPath = os.path.join(self.__outputDir, os.path.basename(outputPath))
command = "mv -f %s %s" % (outputPath, newOutputPath)
logging.debug("Execute command %s" % command)
status, output = commands.getstatusoutput(command)
logging.debug("Status %s output %s" % (status, output))
if status:
toMvFiles.append(line)
else:
handlemv.write(line.replace(outputPath, newOutputPath)+"\n")
if toMvFiles:
logging.error("Failed mv files: %s" % toMvFiles)
handlemv.close()
shutil.copyfile(outputFile + ".mv", os.path.join(self.__outputDir, os.path.basename(outputFile)))
os.rename(outputFile, outputFile + ".BAK")
if yodaReportedFiles:
handleYoda = open(outputFile + ".yodaReported", "w")
for line in yodaStagedFiles:
handleYoda.write(line + "\n")
handleYoda.close()
if os.path.exists(outputFile):
os.rename(outputFile, outputFile + ".BAK")
if yodaStagedFiles:
handleYoda = open(outputFile + ".yodaStaged", "w")
for line in yodaStagedFiles:
handleYoda.write(line + "\n")
handleYoda.close()
handleYoda = open(outputFile + ".yodaStaged.reported", "w")
for chunk in pUtil.chunks(yodaStagedFiles, 100):
try:
eventRanges = []
for outputEvents in chunk:
jobId, eventRangeID, status, output = outputEvents.split(" ")
if status == 'stagedOut' or status == 'reported':
eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'finished'})
if status.startswith("ERR"):
eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'failed'})
update_status, update_output = self.updateEventRanges(eventRanges)
logging.info("update Event Range: status: %s, output: %s" % (update_status, update_output))
if update_status:
update_status, update_output = self.updateEventRanges(eventRanges)
logging.info("update Event retry Range: status: %s, output: %s" % (update_status, update_output))
if update_status == 0:
try:
ret_outputs = json.loads(json.loads(update_output))
if len(ret_outputs) == len(chunk):
for i in range(len(ret_outputs)):
try:
if ret_outputs[i]:
jobId, eventRangeID, status, output = chunk[i].split(" ")
handleYoda.write('{0} {1} {2} {3}\n'.format(jobId, eventRangeID, status, output))
except:
logging.warning("Failed to book updated status %s: %s" % (output, traceback.format_exc()))
except:
logging.warning(traceback.format_exc())
except:
logging.warning(traceback.format_exc())
handleYoda.close()
if os.path.exists(outputFile):
os.rename(outputFile, outputFile + ".BAK")
def processFile(self, outputFile):
handle = open(outputFile)
toMvFiles = []
yodaStagedFiles = []
yodaReportedFiles = []
for line in handle:
line = line.replace(" ", " ")
jobId, eventRange, status, output = line.split(" ")
if status == 'stagedOut' or status.startswith("ERR"):
yodaStagedFiles.append(line)
elif status == 'reported':
yodaReportedFiles.append(line)
yodaStagedFiles.append(line)
else:
toMvFiles.append(line)
handle.close()
if toMvFiles:
handlemv = open(outputFile + ".mv", "w")
for i in range(3):
tmpToMvFiles = toMvFiles
toMvFiles = []
for line in tmpToMvFiles:
line = line.replace(" ", " ")
jobId, eventRange, status, output = line.split(" ")
outputPath = output.split(",")[0]
newOutputPath = os.path.join(self.__outputDir,
os.path.basename(outputPath))
command = "mv -f %s %s" % (outputPath, newOutputPath)
logging.debug("Execute command %s" % command)
status, output = commands.getstatusoutput(command)
logging.debug("Status %s output %s" % (status, output))
if status:
toMvFiles.append(line)
else:
handlemv.write(
line.replace(outputPath, newOutputPath) + "\n")
if toMvFiles:
logging.error("Failed mv files: %s" % toMvFiles)
handlemv.close()
shutil.copyfile(
outputFile + ".mv",
os.path.join(self.__outputDir, os.path.basename(outputFile)))
os.rename(outputFile, outputFile + ".BAK")
if yodaReportedFiles:
handleYoda = open(outputFile + ".yodaReported", "w")
for line in yodaStagedFiles:
handleYoda.write(line + "\n")
handleYoda.close()
if os.path.exists(outputFile):
os.rename(outputFile, outputFile + ".BAK")
if yodaStagedFiles:
handleYoda = open(outputFile + ".yodaStaged", "w")
for line in yodaStagedFiles:
handleYoda.write(line + "\n")
handleYoda.close()
handleYoda = open(outputFile + ".yodaStaged.reported", "w")
for chunk in pUtil.chunks(yodaStagedFiles, 100):
try:
eventRanges = []
for outputEvents in chunk:
jobId, eventRangeID, status, output = outputEvents.split(
" ")
if status == 'stagedOut' or status == 'reported':
eventRanges.append({
"eventRangeID": eventRangeID,
"eventStatus": 'finished'
})
if status.startswith("ERR"):
eventRanges.append({
"eventRangeID": eventRangeID,
"eventStatus": 'failed'
})
update_status, update_output = self.updateEventRanges(
eventRanges)
logging.info("update Event Range: status: %s, output: %s" %
(update_status, update_output))
if update_status:
update_status, update_output = self.updateEventRanges(
eventRanges)
logging.info(
"update Event retry Range: status: %s, output: %s"
% (update_status, update_output))
if update_status == 0:
try:
ret_outputs = json.loads(json.loads(update_output))
if len(ret_outputs) == len(chunk):
for i in range(len(ret_outputs)):
try:
if ret_outputs[i]:
jobId, eventRangeID, status, output = chunk[
i].split(" ")
handleYoda.write(
'{0} {1} {2} {3}\n'.format(
jobId, eventRangeID,
status, output))
except:
logging.warning(
"Failed to book updated status %s: %s"
% (output, traceback.format_exc()))
except:
logging.warning(traceback.format_exc())
except:
logging.warning(traceback.format_exc())
handleYoda.close()
if os.path.exists(outputFile):
os.rename(outputFile, outputFile + ".BAK")
