# loop over JSON files, which will give the list of files to be merged for i in range(0, len(afterString)): if not afterString[i].endswith(".jsn"): continue if "index" in afterString[i]: continue if afterString[i].endswith("recv"): continue if "EoLS" in afterString[i]: continue if "BoLS" in afterString[i]: continue if "EoR" in afterString[i]: continue fileNameString = os.path.basename(afterString[i]).split('_') if(float(debug) >= 50): log.info("FILE: {0}".format(afterString[i])) inputJsonFile = os.path.join(inputDataFolder, afterString[i]) if(float(debug) >= 50): log.info("inputJsonFile: {0}".format(inputJsonFile)) settings = cmsDataFlowMerger.readJsonFile(inputJsonFile,debug) # avoid corrupted files or streamEvD files if("bad" in settings): continue # this is the number of input and output events, and the name of the dat file, something critical # eventsOutput is actually the total number of events to merge in the macromerged stage eventsInput = int(settings['data'][0]) eventsOutput = int(settings['data'][1]) errorCode = 0 file = "" fileErrorString = None fileSize = 0 nFilesBU = 0 eventsTotalInput = 0 checkSum = 0
def doReading(theInput,theMaxTime,theTooSlowTime,theDebug): initReadingTime = time.time() endReadingTime = 0 totalReadFiles = 0 totalTimeFiles = 0 totalTooSlowFiles = 0 while 1: endReadingTime = time.time() diffTime = endReadingTime-initReadingTime if(theMaxTime > 0 and diffTime > theMaxTime): msg = "Maximum time (%f sec) has passed %f sec\n" % (diffTime,theMaxTime) msg += "Average time: %f msec\n" % (totalTimeFiles/totalReadFiles) msg += "Total too slow read files(%f msec): %d out of %d\n" % (theTooSlowTime,totalTooSlowFiles,totalReadFiles) print msg return inputDataFolders = glob.glob(theInput) if(debug >= 0): log.info("***************NEW LOOP***************") if(debug > 0): log.info("inputDataFolders: {0}".format(inputDataFolders)) for nf in range(0, len(inputDataFolders)): inputDataFolder = inputDataFolders[nf] after = dict() listFolders = sorted(glob.glob(os.path.join(inputDataFolder, 'stream*'))); for nStr in range(0, len(listFolders)): try: after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], '*.jsn'))]) after.update(after_temp) after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], '*.ini'))]) after.update(after_temp) after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], 'jsns', '*.jsn'))]) after.update(after_temp) after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], 'data', '*.ini'))]) after.update(after_temp) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format(inputDataFolder,e)) afterStringNoSorted = [f for f in after] afterString = sorted(afterStringNoSorted, reverse=False) for i in range(0, len(afterString)): if not afterString[i].endswith(".jsn"): continue if "index" in afterString[i]: continue if afterString[i].endswith("recv"): continue if "EoLS" in afterString[i]: continue if "BoLS" in afterString[i]: continue if "EoR" in afterString[i]: continue inputJsonFile = os.path.join(inputDataFolder, afterString[i]) initJsonTime = time.time() settings = str(cmsDataFlowMerger.readJsonFile(inputJsonFile, theDebug)) endJsonTime = time.time() # avoid corrupted files if("bad" in settings): continue diffProcessTime = (endJsonTime-initJsonTime)*1000 totalReadFiles = totalReadFiles + 1 totalTimeFiles = totalTimeFiles + diffProcessTime if(diffProcessTime > tooSlowTime): totalTooSlowFiles = totalTooSlowFiles + 1 if(theDebug >= 1): log.info("Time in ms({0}): {1:5.1f}".format(inputJsonFile,diffProcessTime)) endReadingTime = time.time() diffTime = endReadingTime-initReadingTime if(theMaxTime > 0 and diffTime > theMaxTime): break
def doReading(theInput, theMaxTime, theTooSlowTime, theDebug): initReadingTime = time.time() endReadingTime = 0 totalReadFiles = 0 totalTimeFiles = 0 totalTooSlowFiles = 0 while 1: endReadingTime = time.time() diffTime = endReadingTime - initReadingTime if (theMaxTime > 0 and diffTime > theMaxTime): msg = "Maximum time (%f sec) has passed %f sec\n" % (diffTime, theMaxTime) msg += "Average time: %f msec\n" % (totalTimeFiles / totalReadFiles) msg += "Total too slow read files(%f msec): %d out of %d\n" % ( theTooSlowTime, totalTooSlowFiles, totalReadFiles) print msg return inputDataFolders = glob.glob(theInput) if (debug >= 0): log.info("***************NEW LOOP***************") if (debug > 0): log.info("inputDataFolders: {0}".format(inputDataFolders)) for nf in range(0, len(inputDataFolders)): inputDataFolder = inputDataFolders[nf] after = dict() try: after_temp = dict([ (f, None) for f in glob.glob(os.path.join(inputDataFolder, '*.jsn')) ]) after.update(after_temp) after_temp = dict([ (f, None) for f in glob.glob(os.path.join(inputDataFolder, '*.ini')) ]) after.update(after_temp) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format( inputDataFolder, e)) listFolders = sorted( glob.glob(os.path.join(inputDataFolder, 'stream*'))) for nStr in range(0, len(listFolders)): try: after_temp = dict([(f, None) for f in glob.glob( os.path.join(listFolders[nStr], '*.jsn'))]) after.update(after_temp) after_temp = dict([(f, None) for f in glob.glob( os.path.join(listFolders[nStr], '*.ini'))]) after.update(after_temp) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format( inputDataFolder, e)) afterStringNoSorted = [f for f in after] afterString = sorted(afterStringNoSorted, reverse=False) for i in range(0, len(afterString)): if not afterString[i].endswith(".jsn"): continue if "index" in afterString[i]: continue if afterString[i].endswith("recv"): continue if "EoLS" in afterString[i]: continue if "BoLS" in afterString[i]: continue if "EoR" in afterString[i]: continue inputJsonFile = os.path.join(inputDataFolder, afterString[i]) initJsonTime = time.time() settings = str( cmsDataFlowMerger.readJsonFile(inputJsonFile, theDebug)) endJsonTime = time.time() # avoid corrupted files if ("bad" in settings): continue diffProcessTime = (endJsonTime - initJsonTime) * 1000 totalReadFiles = totalReadFiles + 1 totalTimeFiles = totalTimeFiles + diffProcessTime if (diffProcessTime > tooSlowTime): totalTooSlowFiles = totalTooSlowFiles + 1 if (theDebug >= 1): log.info("Time in ms({0}): {1:5.1f}".format( inputJsonFile, diffProcessTime)) endReadingTime = time.time() diffTime = endReadingTime - initReadingTime if (theMaxTime > 0 and diffTime > theMaxTime): break
def cleanUpRun(debug, EoRFileName, inputDataFolder, afterString, path_eol, theRunNumber, outputSMMergedFolder, outputEndName, completeMergingThreshold): settingsEoR = cmsDataFlowMerger.readJsonFile(EoRFileName, debug) if("bad" in settingsEoR): return False eventsInputBU = int(settingsEoR['data'][0]) eventsInputFU = 0 for nb in range(0, len(afterString)): if "EoR" not in afterString[nb]: continue inputEoRFUJsonFile = afterString[nb] settingsLS = cmsDataFlowMerger.readJsonFile(inputEoRFUJsonFile, debug) if("bad" in settingsLS): continue eventsInputFU = eventsInputFU + int(settingsLS['data'][0]) if(float(debug) >= 50): log.info( "eventsInputBU vs. eventsInputFU: {0} vs. {1}".format (eventsInputBU,eventsInputFU)) if (eventsInputBU*completeMergingThreshold <= eventsInputFU): numberBoLSFiles = 0 for nb in range(0, len(afterString)): if not afterString[nb].endswith("_BoLS.jsn"): continue if "DQM" in afterString[nb]: continue if "streamError" in afterString[nb]: continue numberBoLSFiles = numberBoLSFiles + 1 if(float(debug) >= 50): log.info( "numberBoLSFiles: {0}".format(numberBoLSFiles)) EoLSFolder = os.path.join(path_eol, theRunNumber) eventsEoLS = [0, 0, 0] eventsEoLS_noLastLS = [0, 0, 0] lastLumiBU = doSumEoLS(EoLSFolder, eventsEoLS, eventsEoLS_noLastLS) if(eventsEoLS[0] != eventsInputBU): log.info("PROBLEM eventsEoLS != eventsInputBU: {0} vs. {1}".format( eventsEoLS[0],eventsInputBU)) # This is done to make sure there won't be files created after we start deleting the folder deltaTimeHLTFolderCreation = timedelta(minutes=10) hltFolder = os.path.join(EoLSFolder,"hlt") if(os.path.exists(hltFolder)): m_time_stamp = int(os.stat(hltFolder).st_ctime) m_utc_date_time = datetime.utcfromtimestamp(m_time_stamp) deltaTimeHLTFolderCreation = datetime.utcnow() - m_utc_date_time else: log.error("PROBLEM HLT folder does not exist {0}".format(hltFolder)) if(numberBoLSFiles == 0 and eventsInputBU == eventsInputFU and (deltaTimeHLTFolderCreation > timedelta(minutes=5) or lastLumiBU > 3)): # This is needed to cleanUp the macroMerger later EoRFileNameMiniOutput = ( outputSMMergedFolder + "/" + theRunNumber + "_ls0000_MiniEoR_" + outputEndName + ".jsn_TEMP") EoRFileNameMiniOutputStable = ( outputSMMergedFolder + "/" + theRunNumber + "_ls0000_MiniEoR_" + outputEndName + ".jsn") theEoRFileMiniOutput = open(EoRFileNameMiniOutput, 'w') theEoRFileMiniOutput.write( json.dumps({'eventsInputBU': eventsInputBU, 'eventsInputFU': eventsInputFU, 'numberBoLSFiles': numberBoLSFiles, 'eventsTotalRun': eventsEoLS[1], 'eventsLostBU': eventsEoLS[2], 'eventsInputBU_noLastLS': eventsEoLS_noLastLS[0], 'eventsTotalRun_noLastLS': eventsEoLS_noLastLS[1], 'eventsLostBU_noLastLS': eventsEoLS_noLastLS[2], 'lastLumiBU': lastLumiBU})) theEoRFileMiniOutput.close() shutil.move(EoRFileNameMiniOutput, EoRFileNameMiniOutputStable) log.info("Run folder deletion is triggered!: {0} and {1}".format( inputDataFolder,EoLSFolder)) time.sleep(10) ### DEBUG, CHECK FOLDERS TO BE DELETED listFolders = sorted(glob.glob(os.path.join(inputDataFolder, 'stream*'))); after_eor = dict() try: after_temp_eor = dict ([(f, None) for f in glob.glob(os.path.join(inputDataFolder, '*.jsn'))]) after_eor.update(after_temp_eor) for nStr in range(0, len(listFolders)): after_temp_eor = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], '*.jsn'))]) after_eor.update(after_temp_eor) after_temp_eor = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], 'jsns', '*.jsn'))]) after_eor.update(after_temp_eor) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format(inputDataFolder,e)) afterStringNow = [f for f in after_eor] log.info("Checking folders in {0} before deleting them".format(inputDataFolder)) log.info("What we had: {0}".format(afterString)) log.info("What we have: {0}".format(afterStringNow)) ### if(len(afterString) == len(afterStringNow)): try: shutil.rmtree(inputDataFolder) except Exception,e: log.error("Failed removing {0} - {1}".format(inputDataFolder,e)) try: if os.path.islink(EoLSFolder): link_dir = os.readlink(EoLSFolder) log.info("EoLS run dir is a symlink pointing to {0}".format( link_dir)) os.unlink(EoLSFolder) EoLSFolder = link_dir shutil.rmtree(EoLSFolder) except Exception,e: log.error("Failed removing {0} - {1}".format(EoLSFolder,e)) return True
raise RuntimeError, msg after = dict() try: after_temp = dict ([(f, None) for f in glob.glob(os.path.join(inputDataFolder, '*.jsn'))]) after.update(after_temp) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format(inputDataFolder,e)) afterStringNoSorted = [f for f in after if ((dataString in f) and ("EoLS" not in f) and ("BoLS" not in f) and ("EoR" not in f))] afterString = sorted(afterStringNoSorted, reverse=False) dataFiles = [] for i in range(0, len(afterString)): jsonFile = os.path.join(inputDataFolder, afterString[i]) settings = cmsDataFlowMerger.readJsonFile(jsonFile,0) if ("bad" in settings): print "corrupted file: ",jsonFile continue eventsInput = eventsInput + int(settings['data'][0]) eventsInputFiles = eventsInputFiles + 1 if(typeMerging != "mini"): eventsTotalInput = int(settings['data'][7]) fileDataString = afterString[i].split('_') dataFiles.append(fileDataString[3].split('.jsn')[0]) iniFiles = []
def doTheChecking(paths_to_watch, path_eol, mergeType, debug): eventsIDict = dict() eventsEoLSDict = dict() eventsLDict = dict() EoLSProblemDict = dict() eventsFUBUDict = dict() if(float(debug) >= 10): log.info("I will watch: {0}".format(paths_to_watch)) inputDataFoldersNoSorted = glob.glob(paths_to_watch) inputDataFolders = sorted(inputDataFoldersNoSorted, reverse=True) if(float(debug) >= 20): log.info("inputDataFolders: {0}".format(inputDataFolders)) for nf in range(0, len(inputDataFolders)): inputDataFolder = inputDataFolders[nf] inputDataFolderString = inputDataFolder.split('/') # if statement to allow ".../" or ... for input folders if inputDataFolderString[len(inputDataFolderString)-1] == '': theRunNumber = inputDataFolderString[len(inputDataFolderString)-2] else: theRunNumber = inputDataFolderString[len(inputDataFolderString)-1] after = dict() listFolders = sorted(glob.glob(os.path.join(inputDataFolder, 'stream*'))); for nStr in range(0, len(listFolders)): try: after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], '*.jsn'))]) after.update(after_temp) after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], '*.ini'))]) after.update(after_temp) after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], 'jsns', '*.jsn'))]) after.update(after_temp) after_temp = dict ([(f, None) for f in glob.glob(os.path.join(listFolders[nStr], 'data', '*.ini'))]) after.update(after_temp) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format(inputDataFolder,e)) afterStringNoSorted = [f for f in after] afterString = sorted(afterStringNoSorted, reverse=False) if(float(debug) >= 50): log.info("afterString: {0}".format(afterString)) # loop over JSON files, which will give the list of files to be merged for i in range(0, len(afterString)): if not afterString[i].endswith(".jsn"): continue if "index" in afterString[i]: continue if afterString[i].endswith("recv"): continue if "EoLS" in afterString[i]: continue if "BoLS" in afterString[i]: continue if "EoR" in afterString[i]: continue fileNameString = os.path.basename(afterString[i]).split('_') if(float(debug) >= 50): log.info("FILE: {0}".format(afterString[i])) inputJsonFile = os.path.join(inputDataFolder, afterString[i]) if(float(debug) >= 50): log.info("inputJsonFile: {0}".format(inputJsonFile)) settings = cmsDataFlowMerger.readJsonFile(inputJsonFile,debug) # avoid corrupted files or streamEvD files if("bad" in settings): continue # this is the number of input and output events, and the name of the dat file, something critical # eventsOutput is actually the total number of events to merge in the macromerged stage eventsInput = int(settings['data'][0]) eventsOutput = int(settings['data'][1]) errorCode = 0 file = "" fileErrorString = None fileSize = 0 nFilesBU = 0 eventsTotalInput = 0 checkSum = 0 NLostEvents = 0 transferDest = "dummy" if mergeType == "mini": eventsOutputError = int(settings['data'][2]) errorCode = int(settings['data'][3]) file = str(settings['data'][4]) fileSize = int(settings['data'][5]) checkSum = int(settings['data'][7]) # processed events == input + error events eventsInput = eventsInput + eventsOutputError if fileNameString[2] == "streamError": file = str(settings['data'][6]) fileErrorString = file.split(',') else: errorCode = int(settings['data'][2]) file = str(settings['data'][3]) fileSize = int(settings['data'][4]) checkSum = int(settings['data'][5]) nFilesBU = int(settings['data'][6]) eventsTotalInput = int(settings['data'][7]) NLostEvents = int(settings['data'][8]) key = (fileNameString[0],fileNameString[1],fileNameString[2]) if key in eventsIDict.keys(): eventsInput = eventsIDict[key][0] + eventsInput eventsIDict[key].remove(eventsIDict[key][0]) eventsIDict.update({key:[eventsInput]}) NLostEvents = eventsLDict[key][0] + NLostEvents eventsLDict[key].remove(eventsLDict[key][0]) eventsLDict.update({key:[NLostEvents]}) else: eventsIDict.update({key:[eventsInput]}) eventsLDict.update({key:[NLostEvents]}) if mergeType == "mini": keyEoLS = (fileNameString[0],fileNameString[1]) if keyEoLS not in eventsEoLSDict.keys(): EoLSName = path_eol + "/" + fileNameString[0] + "/" + fileNameString[0] + "_" + fileNameString[1] + "_EoLS.jsn" if(float(debug) >= 20): log.info("EoLSName: {0}".format(EoLSName)) if os.path.exists(EoLSName) and os.path.getsize(EoLSName) > 0: inputEoLSName = open(EoLSName, "r").read() settingsEoLS = json.loads(inputEoLSName) eventsEoLS = int(settingsEoLS['data'][0]) filesEoLS = int(settingsEoLS['data'][1]) eventsAllEoLS = int(settingsEoLS['data'][2]) NLostEvents = int(settingsEoLS['data'][3]) eventsEoLSDict.update({keyEoLS:[eventsEoLS,filesEoLS,eventsAllEoLS,NLostEvents]}) else: if (float(debug) >= 30): print "PROBLEM WITH EoLSFile: ",EoLSName keyEoLSRun = (fileNameString[0]) if keyEoLSRun not in EoLSProblemDict.keys(): EoLSProblemDict.update({keyEoLSRun:["bad"]}) if keyEoLS in eventsEoLSDict.keys(): if(float(debug) >= 10): log.info("mini-EventsEoLS/EventsInput-run/ls/stream: {0}, {1} - {2}".format(eventsEoLSDict[keyEoLS][0],eventsIDict[key][0],key)) if(eventsEoLSDict[keyEoLS][0] == eventsIDict[key][0]): if(float(debug) >= 10): log.info("Events match: {0}".format(key)) else: if (float(debug) >= 20): log.info("Events number does not match: EoL says {0} we have in the files: {1}".format(eventsEoLSDict[keyEoLS][0], eventsIDict[key][0])) else: if (float(debug) >= 20): log.warning("Looks like {0} is not in eventsEoLSDict".format(key)) else: if(float(debug) >= 10): log.info("macro-EventsTotalInput/EventsInput/NLostEvents-run/ls/stream: {0}, {1}, {2} - {3}".format(eventsTotalInput,eventsIDict[key][0],eventsLDict[key][0],key)) if(eventsTotalInput == (eventsIDict[key][0]+eventsLDict[key][0])): if(float(debug) >= 10): log.info("Events match: {0}".format(key)) else: if (float(debug) >= 20): log.info("Events number does not match: EoL says {0}, we have in the files: {1} + {2} = {3}".format(eventsTotalInput, eventsIDict[key][0], eventsLDict[key][0], eventsIDict[key][0]+eventsLDict[key][0]))
def cleanUpRun(debug, EoRFileName, inputDataFolder, afterString, path_eol, theRunNumber, outputSMMergedFolder, outputEndName, completeMergingThreshold): settingsEoR = cmsDataFlowMerger.readJsonFile(EoRFileName, debug) if("bad" in settingsEoR): return False eventsInputBU = int(settingsEoR['data'][0]) eventsInputFU = 0 for nb in range(0, len(afterString)): inputEoRFUJsonFile = afterString[nb] settingsLS = cmsDataFlowMerger.readJsonFile(inputEoRFUJsonFile, debug) if("bad" in settingsLS): continue eventsInputFU = eventsInputFU + int(settingsLS['data'][0]) if(float(debug) >= 50): log.info( "eventsInputBU vs. eventsInputFU: {0} vs. {1}".format (eventsInputBU,eventsInputFU)) if (eventsInputBU*completeMergingThreshold <= eventsInputFU): numberBoLSFiles = 0 for nb in range(0, len(afterString)): if not afterString[nb].endswith("_BoLS.jsn"): continue if "DQM" in afterString[nb]: continue if "streamError" in afterString[nb]: continue numberBoLSFiles = numberBoLSFiles + 1 if(float(debug) >= 50): log.info( "numberBoLSFiles: {0}".format(numberBoLSFiles)) EoLSFolder = os.path.join(path_eol, theRunNumber) eventsEoLS = [0, 0, 0] eventsEoLS_noLastLS = [0, 0, 0] lastLumiBU = doSumEoLS(EoLSFolder, eventsEoLS, eventsEoLS_noLastLS) if(eventsEoLS[0] != eventsInputBU): log.info("PROBLEM eventsEoLS != eventsInputBU: {0} vs. {1}".format( eventsEoLS[0],eventsInputBU)) if(numberBoLSFiles == 0 and eventsInputBU == eventsInputFU): # This is needed to cleanUp the macroMerger later EoRFileNameMiniOutput = ( outputSMMergedFolder + "/" + theRunNumber + "_ls0000_MiniEoR_" + outputEndName + ".jsn_TEMP") EoRFileNameMiniOutputStable = ( outputSMMergedFolder + "/" + theRunNumber + "_ls0000_MiniEoR_" + outputEndName + ".jsn") theEoRFileMiniOutput = open(EoRFileNameMiniOutput, 'w') theEoRFileMiniOutput.write( json.dumps({'eventsInputBU': eventsInputBU, 'eventsInputFU': eventsInputFU, 'numberBoLSFiles': numberBoLSFiles, 'eventsTotalRun': eventsEoLS[1], 'eventsLostBU': eventsEoLS[2], 'eventsInputBU_noLastLS': eventsEoLS_noLastLS[0], 'eventsTotalRun_noLastLS': eventsEoLS_noLastLS[1], 'eventsLostBU_noLastLS': eventsEoLS_noLastLS[2], 'lastLumiBU': lastLumiBU})) theEoRFileMiniOutput.close() shutil.move(EoRFileNameMiniOutput, EoRFileNameMiniOutputStable) EoLSFolder = os.path.join(path_eol, theRunNumber) log.info("Run folder deletion is triggered!: {0} and {1}".format( inputDataFolder,EoLSFolder)) time.sleep(10) try: shutil.rmtree(inputDataFolder) except Exception,e: log.error("Failed removing {0} - {1}".format(inputDataFolder,e)) try: if os.path.islink(EoLSFolder): link_dir = os.readlink(EoLSFolder) log.info("EoLS run dir is a symlink pointing to {0}".format( link_dir)) os.unlink(EoLSFolder) EoLSFolder = link_dir shutil.rmtree(EoLSFolder) except Exception,e: log.error("Failed removing {0} - {1}".format(EoLSFolder,e)) return True
]) after.update(after_temp) except Exception, e: log.error("glob.glob operation failed: {0} - {1}".format( inputDataFolder, e)) afterStringNoSorted = [ f for f in after if ((dataString in f) and ("EoLS" not in f) and ( "BoLS" not in f) and ("EoR" not in f)) ] afterString = sorted(afterStringNoSorted, reverse=False) dataFiles = [] for i in range(0, len(afterString)): jsonFile = os.path.join(inputDataFolder, afterString[i]) settings = cmsDataFlowMerger.readJsonFile(jsonFile, 0) if ("bad" in settings): print "corrupted file: ", jsonFile continue eventsInput = eventsInput + int(settings['data'][0]) eventsInputFiles = eventsInputFiles + 1 if (typeMerging != "mini"): eventsTotalInput = int(settings['data'][7]) fileDataString = afterString[i].split('_') dataFiles.append(fileDataString[3].split('.jsn')[0]) iniFiles = []