def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
baseURL = self.config.JobStateMachine.couchurl
dbname = "%s/fwjrs" % getattr(self.config.JobStateMachine, "couchDBName")
self.fwjrAPI = FWJRDBAPI(baseURL, dbname)
self.wmarchiver = WMArchive(self.config.ArchiveDataReporter.WMArchiveURL)
self.numDocsRetrievePerPolling = getattr(self.config.ArchiveDataReporter, "numDocsRetrievePerPolling", 1000)
self.numDocsUploadPerCall = getattr(self.config.ArchiveDataReporter, "numDocsUploadPerCall", 200)
def run(self):
""" Main loop
"""
signal.signal(signal.SIGTERM, self.quit)
logger = logging.getLogger()
logger.info("Starting main loop")
wmarchiver = WMArchive(self.wmarchiveURL, {
'pycurl': True,
"key": self.uploadKey,
"cert": self.uploadCert
})
while not self.stopFlag:
reports = os.listdir(os.path.join(self.baseDir, self.newFjrDir))
currentReps = sorted(reports[:self.bulksize])
logger.debug("Current reports are %s", currentReps)
docs = []
if currentReps:
for rep in currentReps:
repFullname = os.path.join(self.baseDir, self.newFjrDir,
rep)
with open(repFullname) as fd:
#Some params have to be int, see https://github.com/dmwm/CRABServer/issues/5578
#TODO Remove the following 4 lines once we are sure old task are not in the system
tmpdoc = json.load(fd)
for step in tmpdoc["steps"]:
for key in ('NumberOfThreads', 'NumberOfStreams'):
if key in step["performance"]["cpu"]:
step["performance"]["cpu"][key] = int(
float(step["performance"]["cpu"][key]))
for key in ('TotalInitTime', 'TotalInitCPU'):
if key in step["performance"]["cpu"]:
step["performance"]["cpu"][key] = float(
step["performance"]["cpu"][key])
docs.append(tmpdoc)
try:
response = wmarchiver.archiveData(docs)
except (pycurl.error, HTTPException, WMException) as e:
logger.error("Error uploading docs: %s", e)
time.sleep(60)
continue
# Partial success is not allowed either all the insert is successful or none is
if response and response[0]['status'] == "ok" and len(
response[0]['ids']) == len(docs):
logger.info("Successfully uploaded %d docs", len(docs))
for rep in currentReps:
repFullname = os.path.join(self.baseDir,
self.newFjrDir, rep)
repDestName = os.path.join(self.baseDir,
self.processedFjrDir,
self.checkIfFolderExists(),
rep)
os.rename(repFullname, repDestName)
else:
logger.warning(
"Upload failed and it will be retried in the next cycle: %s: %s.",
response[0]['status'], response[0]['reason'])
time.sleep(60)
else:
time.sleep(60)
logger.info("Exiting")
