def mergeFilesA(inpSubFolder, outSubFolder, outputMergedFolder, outputDQMMergedFolder, outputECALMergedFolder, doCheckSum, outMergedFile, outMergedJSON, inputDataFolder, infoEoLS, eventsO, files, checkSum, fileSize, filesJSON, errorCode, transferDest, mergeType, doRemoveFiles, outputEndName, esServerUrl, esIndexName, debug):
try:
if(float(debug) >= 10): log.info("mergeFiles: {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}".format(outputMergedFolder, outMergedFile, outMergedJSON, inputDataFolder, infoEoLS, eventsO, files, checkSum, fileSize, filesJSON, errorCode))
outMergedFileFullPath = os.path.join(outputMergedFolder, outSubFolder, "open", outMergedFile)
outMergedJSONFullPath = os.path.join(outputMergedFolder, outSubFolder, "open", outMergedJSON)
if(float(debug) >= 10): log.info('outMergedFileFullPath: {0}'.format(outMergedFileFullPath))
timeReadWrite = [0, 0]
initMergingTime = time.time()
if(float(debug) > 0): log.info("Start merge of {0}".format(outMergedJSONFullPath))
if os.path.exists(outMergedFileFullPath):
os.remove(outMergedFileFullPath)
if os.path.exists(outMergedJSONFullPath):
os.remove(outMergedJSONFullPath)
inputJsonFolder = os.path.dirname(filesJSON[0])
fileNameString = filesJSON[0].replace(inputJsonFolder,"").replace("/","").split('_')
specialStreams = False
if(fileNameString[2] == "streamDQMHistograms" or fileNameString[2] == "streamHLTRates" or fileNameString[2] == "streamL1Rates"):
specialStreams = True
if (mergeType == "macro" and specialStreams == False and infoEoLS[0] != 0):
iniName = fileNameString[0] + "_ls0000_" + fileNameString[2] + "_" + "StorageManager" + ".ini"
iniNameFullPath = os.path.join(outputMergedFolder, outSubFolder, "open", iniName)
if os.path.exists(iniNameFullPath):
checkSumIni=1
n_retries=0
while n_retries < max_retries:
try:
checkSumIni=1
with open(iniNameFullPath, 'r') as fsrc:
length=16*1024
while 1:
buf = fsrc.read(length)
if not buf:
break
checkSumIni=zlib.adler32(buf,checkSumIni)
checkSumIni = checkSumIni & 0xffffffff
checkSum = zlibextras.adler32_combine(checkSumIni,checkSum,fileSize)
checkSum = checkSum & 0xffffffff
fileSize = os.path.getsize(iniNameFullPath) + fileSize
filenames = [iniNameFullPath]
with open(outMergedFileFullPath, 'w') as fout:
append_files(filenames, fout, debug, timeReadWrite)
fout.close()
break
except Exception, e:
log.warning("Error writing file {0}: {1}, waiting for 30secs".format(outMergedFileFullPath,e))
n_retries+=1
time.sleep(30)
if(n_retries == max_retries):
log.error("Could not write file {0}!: {0}".format(outMergedFileFullPath))
msg = "Could not write file {0}!: %s" % (outMergedFileFullPath)
raise RuntimeError, msg
else:
log.error("BIG PROBLEM, ini file not found!: {0}".format(iniNameFullPath))
msg = "BIG PROBLEM, ini file not found!: %s" % (iniNameFullPath)
raise RuntimeError, msg
filenames = [inputDataFolder + "/" + inpSubFolder + "/" + word_in_list for word_in_list in files]
if(float(debug) > 5): log.info("Will merge: {0}".format(filenames))
if(infoEoLS[0] != 0):
if (specialStreams == False and mergeType == "macro"):
with open(outMergedFileFullPath, 'a') as fout:
append_files(filenames, fout, debug, timeReadWrite)
fout.close()
elif (specialStreams == False):
n_retries=0
while n_retries < max_retries:
try:
with open(outMergedFileFullPath, 'w') as fout:
append_files(filenames, fout, debug, timeReadWrite)
fout.close()
break
except Exception, e:
log.warning("Error writing file {0}: {1}, waiting for 30secs".format(outMergedFileFullPath,e))
n_retries+=1
time.sleep(30)
if(n_retries == max_retries):
log.error("Could not write file {0}!: {0}".format(outMergedFileFullPath))
msg = "Could not write file {0}!: %s" % (outMergedFileFullPath)
raise RuntimeError, msg
elif (fileNameString[2] == "streamHLTRates" or fileNameString[2] == "streamL1Rates"):
msg = "jsonMerger %s " % (outMergedFileFullPath)
goodFiles = 0
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile]) and (not os.path.isdir(filenames[nfile])) and os.path.getsize(filenames[nfile]) > 0):
msg = msg + filenames[nfile] + " "
goodFiles = goodFiles + 1
if(float(debug) > 20): log.info("running {0}".format(msg))
if(goodFiles > 0):
os.system(msg)
else:
open(outMergedFileFullPath, 'w').close()
else:
if (mergeType == "macro"):
msg = "fastHadd add -j 7 -o %s " % (outMergedFileFullPath)
else:
msg = "fastHadd add -o %s " % (outMergedFileFullPath)
goodFiles = 0
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile]) and (not os.path.isdir(filenames[nfile])) and os.path.getsize(filenames[nfile]) > 0):
msg = msg + filenames[nfile] + " "
goodFiles = goodFiles + 1
if(float(debug) > 20): log.info("running {0}".format(msg))
if(goodFiles > 0):
os.system(msg)
try:
fileSize = os.path.getsize(outMergedFileFullPath)
except:
pass
else:
open(outMergedFileFullPath, 'w').close()
# also want to move the lock file to the bad area
lockNameFullPathStable = os.path.join(outputSMMergedFolder, outSubFolder, "bad", lockName)
shutil.move(lockNameFullPath,lockNameFullPathStable)
else:
outMergedFileFullPathStable = os.path.join(outputSMMergedFolder, outSubFolder, outMergedFile)
if(fileNameString[2] != "streamError"):
# expected checksum
with open(lockNameFullPath, 'r+w') as filelock:
lockFullString = filelock.readline().split(',')
checkSum = int(lockFullString[0].split(':')[1])
for nf in range(1, len(lockFullString)):
fileSizeAux = int(lockFullString[nf].split(':')[0].split('=')[1])-int(lockFullString[nf-1].split(':')[0].split('=')[1])
checkSumAux = int(lockFullString[nf].split(':')[1])
checkSum = zlibextras.adler32_combine(checkSum,checkSumAux,fileSizeAux)
checkSum = checkSum & 0xffffffff
if(doCheckSum == "True" and fileNameString[2] != "streamError"):
# observed checksum
adler32c=1
with open(outMergedFileFullPath, 'r') as fsrc:
length=16*1024
while 1:
buf = fsrc.read(length)
if not buf:
break
adler32c=zlib.adler32(buf,adler32c)
adler32c = adler32c & 0xffffffff
else:
outMergedFileFullPathStable = os.path.join(
outputSMMergedFolder, outSubFolder, outMergedFile)
if (fileNameString[2] != "streamError"):
# expected checksum
with open(lockNameFullPath, 'r+w') as filelock:
lockFullString = filelock.readline().split(',')
checkSum = int(lockFullString[0].split(':')[1])
for nf in range(1, len(lockFullString)):
fileSizeAux = int(lockFullString[nf].split(
':')[0].split('=')[1]) - int(
lockFullString[nf -
1].split(':')[0].split('=')[1])
checkSumAux = int(lockFullString[nf].split(':')[1])
checkSum = zlibextras.adler32_combine(
checkSum, checkSumAux, fileSizeAux)
checkSum = checkSum & 0xffffffff
if (doCheckSum == "True"
and fileNameString[2] != "streamError"):
# observed checksum
adler32c = 1
with open(outMergedFileFullPath, 'r') as fsrc:
length = 16 * 1024
while 1:
buf = fsrc.read(length)
if not buf:
break
adler32c = zlib.adler32(buf, adler32c)
adler32c = adler32c & 0xffffffff
def mergeFilesA(inpSubFolder, outSubFolder, outputMergedFolder,
outputDQMMergedFolder, outputECALMergedFolder, doCheckSum,
outMergedFile, outMergedJSON, inputDataFolder, infoEoLS,
eventsO, files, checkSum, fileSize, filesJSON, errorCode,
transferDest, mergeType, doRemoveFiles, outputEndName,
esServerUrl, esIndexName, debug):
try:
if (float(debug) >= 10):
log.info(
"mergeFiles: {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}".
format(outputMergedFolder, outMergedFile, outMergedJSON,
inputDataFolder, infoEoLS, eventsO, files, checkSum,
fileSize, filesJSON, errorCode))
outMergedFileFullPath = os.path.join(outputMergedFolder, outSubFolder,
"open", outMergedFile)
outMergedJSONFullPath = os.path.join(outputMergedFolder, outSubFolder,
"open", outMergedJSON)
if (float(debug) >= 10):
log.info(
'outMergedFileFullPath: {0}'.format(outMergedFileFullPath))
timeReadWrite = [0, 0]
initMergingTime = time.time()
if (float(debug) > 0):
log.info("Start merge of {0}".format(outMergedJSONFullPath))
if os.path.exists(outMergedFileFullPath):
os.remove(outMergedFileFullPath)
if os.path.exists(outMergedJSONFullPath):
os.remove(outMergedJSONFullPath)
inputJsonFolder = os.path.dirname(filesJSON[0])
fileNameString = filesJSON[0].replace(inputJsonFolder,
"").replace("/", "").split('_')
specialStreams = False
if (fileNameString[2] == "streamDQMHistograms"
or fileNameString[2] == "streamHLTRates"
or fileNameString[2] == "streamL1Rates"):
specialStreams = True
if (mergeType == "macro" and specialStreams == False
and infoEoLS[0] != 0):
iniName = fileNameString[0] + "_ls0000_" + fileNameString[
2] + "_" + "StorageManager" + ".ini"
iniNameFullPath = os.path.join(outputMergedFolder, outSubFolder,
"open", iniName)
if os.path.exists(iniNameFullPath):
checkSumIni = 1
n_retries = 0
while n_retries < max_retries:
try:
checkSumIni = 1
with open(iniNameFullPath, 'r') as fsrc:
length = 16 * 1024
while 1:
buf = fsrc.read(length)
if not buf:
break
checkSumIni = zlib.adler32(buf, checkSumIni)
checkSumIni = checkSumIni & 0xffffffff
checkSum = zlibextras.adler32_combine(
checkSumIni, checkSum, fileSize)
checkSum = checkSum & 0xffffffff
fileSize = os.path.getsize(iniNameFullPath) + fileSize
filenames = [iniNameFullPath]
with open(outMergedFileFullPath, 'w') as fout:
append_files(filenames, fout, debug, timeReadWrite)
fout.close()
break
except Exception, e:
log.warning(
"Error writing file {0}: {1}, waiting for 30secs".
format(outMergedFileFullPath, e))
n_retries += 1
time.sleep(30)
if (n_retries == max_retries):
log.error("Could not write file {0}!: {0}".format(
outMergedFileFullPath))
msg = "Could not write file {0}!: %s" % (
outMergedFileFullPath)
raise RuntimeError, msg
else:
log.error("BIG PROBLEM, ini file not found!: {0}".format(
iniNameFullPath))
msg = "BIG PROBLEM, ini file not found!: %s" % (
iniNameFullPath)
raise RuntimeError, msg
filenames = [
inputDataFolder + "/" + inpSubFolder + "/" + word_in_list
for word_in_list in files
]
if (float(debug) > 5): log.info("Will merge: {0}".format(filenames))
if (infoEoLS[0] != 0):
if (specialStreams == False and mergeType == "macro"):
with open(outMergedFileFullPath, 'a') as fout:
append_files(filenames, fout, debug, timeReadWrite)
fout.close()
elif (specialStreams == False):
n_retries = 0
while n_retries < max_retries:
try:
with open(outMergedFileFullPath, 'w') as fout:
append_files(filenames, fout, debug, timeReadWrite)
fout.close()
break
except Exception, e:
log.warning(
"Error writing file {0}: {1}, waiting for 30secs".
format(outMergedFileFullPath, e))
n_retries += 1
time.sleep(30)
if (n_retries == max_retries):
log.error("Could not write file {0}!: {0}".format(
outMergedFileFullPath))
msg = "Could not write file {0}!: %s" % (
outMergedFileFullPath)
raise RuntimeError, msg
elif (fileNameString[2] == "streamHLTRates"
or fileNameString[2] == "streamL1Rates"):
msg = "jsonMerger %s " % (outMergedFileFullPath)
goodFiles = 0
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile])
and (not os.path.isdir(filenames[nfile]))
and os.path.getsize(filenames[nfile]) > 0):
msg = msg + filenames[nfile] + " "
goodFiles = goodFiles + 1
if (float(debug) > 20): log.info("running {0}".format(msg))
if (goodFiles > 0):
os.system(msg)
else:
open(outMergedFileFullPath, 'w').close()
else:
if (mergeType == "macro"):
msg = "fastHadd add -j 7 -o %s " % (outMergedFileFullPath)
else:
msg = "fastHadd add -o %s " % (outMergedFileFullPath)
goodFiles = 0
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile])
and (not os.path.isdir(filenames[nfile]))
and os.path.getsize(filenames[nfile]) > 0):
msg = msg + filenames[nfile] + " "
goodFiles = goodFiles + 1
if (float(debug) > 20): log.info("running {0}".format(msg))
if (goodFiles > 0):
os.system(msg)
try:
fileSize = os.path.getsize(outMergedFileFullPath)
except:
pass
else:
open(outMergedFileFullPath, 'w').close()
def mergeFilesA(inpSubFolder, outSubFolder, outputMergedFolder, outputDQMMergedFolder, doCheckSum, outMergedFile, outMergedJSON, inputDataFolder, infoEoLS, eventsO, files, checkSumFix, fileSizeFix, filesJSON, errorCode, transferDest, timeReadJsns, timeIni, timeEndJsonOps, mergeType, doRemoveFiles, outputEndName, esServerUrl, esIndexName, debug):
try:
if(float(debug) >= 10): log.info("mergeFiles: {0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}".format(outputMergedFolder, outMergedFile, outMergedJSON, inputDataFolder, infoEoLS, eventsO, files, checkSumFix, fileSizeFix, filesJSON, errorCode))
outMergedFileFullPath = os.path.join(outputMergedFolder, outSubFolder, "data", "open", outMergedFile)
outMergedJSONFullPath = os.path.join(outputMergedFolder, outSubFolder, "jsns", "open", outMergedJSON)
if(float(debug) >= 10): log.info('outMergedFileFullPath: {0}'.format(outMergedFileFullPath))
outMergedFileFullPathStable = ""
outMergedJSONFullPathStable = ""
paramTimeOut = [False, False] # 1: is/isn't time out? / 2: want to check it?
timeReadWrite = [0, 0]
initMergingTime = time.time()
if(float(debug) > 0): log.info("Start merge of {0}".format(outMergedJSONFullPath))
inputJsonFolder = os.path.dirname(filesJSON[0])
fileNameString = filesJSON[0].replace(inputJsonFolder,"").replace("/","").split('_')
if("DQM" in fileNameString[2] and mergeType == "macro"): paramTimeOut[1] = True; # check timeout only for DQM streams
specialStreams = False
if(fileNameString[2] == "streamDQMHistograms" or fileNameString[2] == "streamHLTRates" or fileNameString[2] == "streamL1Rates" or fileNameString[2] == "streamError"):
specialStreams = True
checksum_status = False
number_full_tries = 0
while(checksum_status == False and number_full_tries < 3):
checksum_status = True
number_full_tries = number_full_tries + 1
if os.path.exists(outMergedFileFullPath):
os.remove(outMergedFileFullPath)
if os.path.exists(outMergedJSONFullPath):
os.remove(outMergedJSONFullPath)
checkSumExpected = checkSumFix
fileSizeExpected = fileSizeFix
checkSumRead = [1]
if (mergeType == "macro" and specialStreams == False and infoEoLS[0] != 0):
iniName = fileNameString[0] + "_ls0000_" + fileNameString[2] + "_" + "StorageManager" + ".ini"
iniNameFullPath = os.path.join(outputMergedFolder, outSubFolder, "data", "open", iniName)
if os.path.exists(iniNameFullPath):
checkSumIni = 1
n_retries=0
while n_retries < max_retries:
if(n_retries > 0 and paramTimeOut[1] == True): paramTimeOut[0] = True
try:
checkSumIni=1
with open(iniNameFullPath, 'r') as fsrc:
length=16*1024
while 1:
buf = fsrc.read(length)
if not buf:
break
checkSumIni=zlib.adler32(buf,checkSumIni)
fsrc.close()
checkSumIni = checkSumIni & 0xffffffff
checkSumExpected = zlibextras.adler32_combine(checkSumIni,checkSumFix,fileSizeFix)
checkSumExpected = checkSumExpected & 0xffffffff
fileSizeExpected = os.path.getsize(iniNameFullPath) + fileSizeFix
filenames = [iniNameFullPath]
with open(outMergedFileFullPath, 'w') as fout:
append_files(filenames, fout, checkSumRead, debug, timeReadWrite, paramTimeOut)
fout.close()
break
except Exception, e:
log.warning("Error writing file {0}: {1}, waiting for 30secs".format(outMergedFileFullPath,e))
n_retries+=1
time.sleep(30)
if(n_retries == max_retries):
log.error("Could not write file {0}!".format(outMergedFileFullPath))
msg = "Could not write file {0}!: %s" % (outMergedFileFullPath)
raise RuntimeError, msg
else:
log.error("BIG PROBLEM, ini file not found!: {0}".format(iniNameFullPath))
msg = "BIG PROBLEM, ini file not found!: %s" % (iniNameFullPath)
raise RuntimeError, msg
filenames = [inputDataFolder + "/" + inpSubFolder + "/data/" + word_in_list for word_in_list in files]
if(float(debug) > 5): log.info("Will merge: {0}".format(filenames))
if(infoEoLS[0] != 0):
if (specialStreams == False and mergeType == "macro"):
with open(outMergedFileFullPath, 'a') as fout:
append_files(filenames, fout, checkSumRead, debug, timeReadWrite, paramTimeOut)
fout.close()
elif (specialStreams == False):
n_retries=0
while n_retries < max_retries:
if(n_retries > 0 and paramTimeOut[1] == True): paramTimeOut[0] = True
try:
with open(outMergedFileFullPath, 'w') as fout:
append_files(filenames, fout, checkSumRead, debug, timeReadWrite, paramTimeOut)
fout.close()
break
except Exception, e:
log.warning("Error writing file {0}: {1}, waiting for 30secs".format(outMergedFileFullPath,e))
n_retries+=1
time.sleep(30)
if(n_retries == max_retries):
log.error("Could not write file {0}!".format(outMergedFileFullPath))
msg = "Could not write file {0}!: %s" % (outMergedFileFullPath)
raise RuntimeError, msg
elif (fileNameString[2] == "streamHLTRates" or fileNameString[2] == "streamL1Rates"):
msg = "jsonMerger %s " % (outMergedFileFullPath)
goodFiles = 0
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile]) and (not os.path.isdir(filenames[nfile])) and os.path.getsize(filenames[nfile]) > 0):
msg = msg + filenames[nfile] + " "
goodFiles = goodFiles + 1
if(float(debug) > 20): log.info("running {0}".format(msg))
if(goodFiles > 0):
os.system(msg)
else:
open(outMergedFileFullPath, 'w').close()
elif (fileNameString[2] == "streamError"):
if (mergeType == "mini"):
filenames = [inputDataFolder + "/" + word_in_list for word_in_list in files]
filenamesOut = [os.path.join(outputMergedFolder, outSubFolder, "data") + "/" + word_in_list for word_in_list in files]
outMergedFile = ""
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile]) and (not os.path.isdir(filenames[nfile])) and os.path.getsize(filenames[nfile]) > 0):
outMergedFile = files[nfile] + "," + outMergedFile
shutil.move(filenames[nfile],filenamesOut[nfile])
else:
if (mergeType == "macro"):
msg = "fastHadd add -j 7 -o %s " % (outMergedFileFullPath)
else:
msg = "fastHadd add -o %s " % (outMergedFileFullPath)
goodFiles = 0
for nfile in range(0, len(filenames)):
if (os.path.exists(filenames[nfile]) and (not os.path.isdir(filenames[nfile])) and os.path.getsize(filenames[nfile]) > 0):
msg = msg + filenames[nfile] + " "
goodFiles = goodFiles + 1
if(float(debug) > 20): log.info("running {0}".format(msg))
if(goodFiles > 0):
os.system(msg)
try:
fileSizeExpected = os.path.getsize(outMergedFileFullPath)
except Exception, e:
log.warning("Error computing histogram filesize {0}: {1}".format(outMergedFileFullPath,e))
else:
