Esempio n. 1
0
    def checkFinishedStagingFiles(self):
        finishedFiles = []
        for file in self.__eventRanges:
            try:
                if len(self.__eventRanges[file].keys()) == 0:
                    filepath = os.path.join(self.__workDir, file)
                    handle = open(filepath + ".staged.reported", 'w')
                    finishedEventRanges = []
                    for chunk in pUtil.chunks(self.__eventRanges_staged[file], 100):
                        try:
                            eventRanges = []
                            for outputEvents in chunk:
                                jobId, eventRangeID, status, output = outputEvents
                                if eventRangeID not in finishedEventRanges:
                                    finishedEventRanges.append(eventRangeID)
                                    if status == 'finished':
                                        eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": status})
                                    if status.startswith("ERR"):
                                        eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'failed'})

                            update_status, update_output = self.updateEventRanges(eventRanges)
                            logging.info("update Event Range: status: %s, output: %s" % (update_status, update_output))
                            if update_status:
                                update_status, update_output = self.updateEventRanges(eventRanges)
                                logging.info("update Event retry Range: status: %s, output: %s" % (update_status, update_output))
                            if update_status == 0:
                                try:
                                    ret_outputs = json.loads(json.loads(update_output))
                                    if len(ret_outputs) == len(chunk):
                                        for i in range(len(ret_outputs)):
                                            try:
                                                if ret_outputs[i]:
                                                    jobId, eventRangeID, status, output = chunk[i]
                                                    logging.info("Remove %s" % output)
                                                    os.remove(output)
                                                    handle.write('{0} {1} {2} {3}\n'.format(jobId, eventRangeID, status, output))

                                            except:
                                                logging.warning("Failed to remove %s: %s" % (output, traceback.format_exc()))
                                except:
                                    logging.warning(traceback.format_exc())
                        except:
                            logging.warning(traceback.format_exc())
                    handle.close()
                    os.rename(filepath + ".staging", filepath + ".BAK")
                    finishedFiles.append(file)
            except:
                logging.warning(traceback.format_exc())
        for file in finishedFiles:
            del self.__eventRanges[file]
            del self.__eventRanges_staged[file]
            del self.__eventRanges_faileStaged[file]
Esempio n. 2
0
    def processFile(self, outputFile):
        handle = open(outputFile)
        toMvFiles = []
        yodaStagedFiles = []
        yodaReportedFiles = []
        for line in handle:
            line = line.replace("  ", " ")
            jobId, eventRange, status, output = line.split(" ")
            if status == 'stagedOut' or status.startswith("ERR"):
                yodaStagedFiles.append(line)
            elif status == 'reported':
                yodaReportedFiles.append(line)
                yodaStagedFiles.append(line)
            else:
                toMvFiles.append(line)
        handle.close()

        if toMvFiles:
            handlemv = open(outputFile + ".mv", "w")
            for i in range(3):
                tmpToMvFiles = toMvFiles
                toMvFiles = []
                for line in tmpToMvFiles:
                    line = line.replace("  ", " ")
                    jobId, eventRange, status, output = line.split(" ")
                    outputPath = output.split(",")[0]
                    newOutputPath = os.path.join(self.__outputDir, os.path.basename(outputPath))
                    command = "mv -f %s %s" % (outputPath, newOutputPath)
                    logging.debug("Execute command %s" % command)
                    status, output = commands.getstatusoutput(command)
                    logging.debug("Status %s output %s" % (status, output))
                    if status:
                        toMvFiles.append(line)
                    else:
                        handlemv.write(line.replace(outputPath, newOutputPath)+"\n")
            if toMvFiles:
                logging.error("Failed mv files: %s" % toMvFiles)
            handlemv.close()
            shutil.copyfile(outputFile + ".mv", os.path.join(self.__outputDir, os.path.basename(outputFile)))
            os.rename(outputFile, outputFile + ".BAK")

        if yodaReportedFiles:
            handleYoda = open(outputFile + ".yodaReported", "w")
            for line in yodaStagedFiles:
                handleYoda.write(line + "\n")
            handleYoda.close()
            if os.path.exists(outputFile):
                os.rename(outputFile, outputFile + ".BAK")

        if yodaStagedFiles:
            handleYoda = open(outputFile + ".yodaStaged", "w")
            for line in yodaStagedFiles:
                handleYoda.write(line + "\n")
            handleYoda.close()
            handleYoda = open(outputFile + ".yodaStaged.reported", "w")
            for chunk in pUtil.chunks(yodaStagedFiles, 100):
                try:
                    eventRanges = []
                    for outputEvents in chunk:
                        jobId, eventRangeID, status, output = outputEvents.split(" ")
                        if status == 'stagedOut' or status == 'reported':
                            eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'finished'})
                        if status.startswith("ERR"):
                            eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'failed'})

                    update_status, update_output = self.updateEventRanges(eventRanges)
                    logging.info("update Event Range: status: %s, output: %s" % (update_status, update_output))
                    if update_status:
                        update_status, update_output = self.updateEventRanges(eventRanges)
                        logging.info("update Event retry Range: status: %s, output: %s" % (update_status, update_output))
                    if update_status == 0:
                        try:
                            ret_outputs = json.loads(json.loads(update_output))
                            if len(ret_outputs) == len(chunk):
                                for i in range(len(ret_outputs)):
                                    try:
                                        if ret_outputs[i]:
                                            jobId, eventRangeID, status, output = chunk[i].split(" ")
                                            handleYoda.write('{0} {1} {2} {3}\n'.format(jobId, eventRangeID, status, output))
                                    except:
                                        logging.warning("Failed to book updated status %s: %s" % (output, traceback.format_exc()))
                        except:
                                    logging.warning(traceback.format_exc())
                except:
                    logging.warning(traceback.format_exc())
            handleYoda.close()
            if os.path.exists(outputFile):
                os.rename(outputFile, outputFile + ".BAK")
Esempio n. 3
0
    def processFile(self, outputFile):
        handle = open(outputFile)
        toMvFiles = []
        yodaStagedFiles = []
        yodaReportedFiles = []
        for line in handle:
            line = line.replace("  ", " ")
            jobId, eventRange, status, output = line.split(" ")
            if status == 'stagedOut' or status.startswith("ERR"):
                yodaStagedFiles.append(line)
            elif status == 'reported':
                yodaReportedFiles.append(line)
                yodaStagedFiles.append(line)
            else:
                toMvFiles.append(line)
        handle.close()

        if toMvFiles:
            handlemv = open(outputFile + ".mv", "w")
            for i in range(3):
                tmpToMvFiles = toMvFiles
                toMvFiles = []
                for line in tmpToMvFiles:
                    line = line.replace("  ", " ")
                    jobId, eventRange, status, output = line.split(" ")
                    outputPath = output.split(",")[0]
                    newOutputPath = os.path.join(self.__outputDir,
                                                 os.path.basename(outputPath))
                    command = "mv -f %s %s" % (outputPath, newOutputPath)
                    logging.debug("Execute command %s" % command)
                    status, output = commands.getstatusoutput(command)
                    logging.debug("Status %s output %s" % (status, output))
                    if status:
                        toMvFiles.append(line)
                    else:
                        handlemv.write(
                            line.replace(outputPath, newOutputPath) + "\n")
            if toMvFiles:
                logging.error("Failed mv files: %s" % toMvFiles)
            handlemv.close()
            shutil.copyfile(
                outputFile + ".mv",
                os.path.join(self.__outputDir, os.path.basename(outputFile)))
            os.rename(outputFile, outputFile + ".BAK")

        if yodaReportedFiles:
            handleYoda = open(outputFile + ".yodaReported", "w")
            for line in yodaStagedFiles:
                handleYoda.write(line + "\n")
            handleYoda.close()
            if os.path.exists(outputFile):
                os.rename(outputFile, outputFile + ".BAK")

        if yodaStagedFiles:
            handleYoda = open(outputFile + ".yodaStaged", "w")
            for line in yodaStagedFiles:
                handleYoda.write(line + "\n")
            handleYoda.close()
            handleYoda = open(outputFile + ".yodaStaged.reported", "w")
            for chunk in pUtil.chunks(yodaStagedFiles, 100):
                try:
                    eventRanges = []
                    for outputEvents in chunk:
                        jobId, eventRangeID, status, output = outputEvents.split(
                            " ")
                        if status == 'stagedOut' or status == 'reported':
                            eventRanges.append({
                                "eventRangeID": eventRangeID,
                                "eventStatus": 'finished'
                            })
                        if status.startswith("ERR"):
                            eventRanges.append({
                                "eventRangeID": eventRangeID,
                                "eventStatus": 'failed'
                            })

                    update_status, update_output = self.updateEventRanges(
                        eventRanges)
                    logging.info("update Event Range: status: %s, output: %s" %
                                 (update_status, update_output))
                    if update_status:
                        update_status, update_output = self.updateEventRanges(
                            eventRanges)
                        logging.info(
                            "update Event retry Range: status: %s, output: %s"
                            % (update_status, update_output))
                    if update_status == 0:
                        try:
                            ret_outputs = json.loads(json.loads(update_output))
                            if len(ret_outputs) == len(chunk):
                                for i in range(len(ret_outputs)):
                                    try:
                                        if ret_outputs[i]:
                                            jobId, eventRangeID, status, output = chunk[
                                                i].split(" ")
                                            handleYoda.write(
                                                '{0} {1} {2} {3}\n'.format(
                                                    jobId, eventRangeID,
                                                    status, output))
                                    except:
                                        logging.warning(
                                            "Failed to book updated status %s: %s"
                                            % (output, traceback.format_exc()))
                        except:
                            logging.warning(traceback.format_exc())
                except:
                    logging.warning(traceback.format_exc())
            handleYoda.close()
            if os.path.exists(outputFile):
                os.rename(outputFile, outputFile + ".BAK")