def testConvertToArchiverFormat(self): job = {} job["id"] = "1-0" job['doc'] = { "fwjr": SAMPLE_FWJR, "jobtype": "Processing", "jobstate": "success", "timestamp": int(time.time()) } newData = createArchiverDoc(job) from pprint import pprint pprint(newData) #outputModules = set([a['outputModule'] for a in newData['steps']['cmsRun1']['output']]) #outModules = set(SAMPLE_FWJR['steps']['cmsRun1']['output'].keys()) #self.assertEqual(outputModules - outModules, set()) run = SAMPLE_FWJR['steps']['cmsRun1']['output'][ 'ALCARECOStreamMuAlCalIsolatedMu'][0]['runs'] for step in newData['steps']: if step['name'] == 'cmsRun1': runInfo = step['output'][0]['runs'][0] self.assertEqual((run[str(runInfo['runNumber'])]), runInfo['lumis']) fwjrSamples = [ "ErrorCodeFail.json", "FailedByAgent.json", "HarvestSuccessFwjr.json", "LogCollectFailedFwjr.json", "LogCollectSuccessFwjr.json", "MergeFailedFwjr.json", "MergeSuccessFwjr.json", "NoJobReportFail.json", "ProcessingFailedFwjr.json", "ProcessingPerformanceFailed.json", "ProcessingSuccessFwjr.json", "ProductionFailedFwjr.json", "ProductionSuccessFwjr.json", "SkimSuccessFwjr.json" ] for sample in fwjrSamples: sPath = os.path.join( getTestBase(), "WMCore_t/Services_t/WMArchive_t/FWJRSamples/%s" % sample) with open(sPath, 'r') as infile: fwjr = json.load(infile) job = {} job["id"] = fwjr["_id"] job['doc'] = { "fwjr": fwjr["fwjr"], "jobtype": fwjr["jobtype"], "jobstate": fwjr['jobstate'], "timestamp": fwjr["timestamp"] } newData = createArchiverDoc(job) print("\n\n==========\n%s" % sPath) pprint(newData)
def algorithm(self, parameters): """ get information from wmbs, workqueue and local couch """ try: data = self.fwjrAPI.getFWJRByArchiveStatus('ready', limit=self.numDocsRetrievePerPolling)['rows'] logging.info("Found %i not archived documents from FWRJ db to upload to WMArchive.", len(data)) for slicedData in grouper(data, self.numDocsUploadPerCall): jobIDs = [] archiveDocs = [] for job in slicedData: doc = createArchiverDoc(job) archiveDocs.append(doc) jobIDs.append(job["id"]) response = self.wmarchiver.archiveData(archiveDocs) # Partial success is not allowed either all the insert is successful or none is if response[0]['status'] == "ok" and len(response[0]['ids']) == len(jobIDs): archiveIDs = response[0]['ids'] for docID in jobIDs: self.fwjrAPI.updateArchiveUploadedStatus(docID) logging.info("...successfully uploaded %d docs", len(jobIDs)) logging.debug("JobIDs uploaded: %s", jobIDs) logging.debug("Archived IDs returned: %s", archiveIDs) else: logging.warning("Upload failed and it will be retried in the next cycle: %s: %s.", response[0]['status'], response[0]['reason']) logging.debug("failed JobIds %s", jobIDs) except Exception as ex: logging.error("Error occurred, will retry later:") logging.error(str(ex)) logging.error("Trace back: \n%s", traceback.format_exc())
def testConvertToArchiverFormat(self): job = {} job["id"] = "1-0" job['doc'] = {"fwjr": SAMPLE_FWJR, "jobtype": "Processing", "jobstate": "success", "timestamp": int(time.time())} newData = createArchiverDoc(job) from pprint import pprint pprint(newData) #outputModules = set([a['outputModule'] for a in newData['steps']['cmsRun1']['output']]) #outModules = set(SAMPLE_FWJR['steps']['cmsRun1']['output'].keys()) #self.assertEqual(outputModules - outModules, set()) run = SAMPLE_FWJR['steps']['cmsRun1']['output']['ALCARECOStreamMuAlCalIsolatedMu'][0]['runs'] for step in newData['steps']: if step['name'] == 'cmsRun1': runInfo = step['output'][0]['runs'][0] self.assertEqual((run[str(runInfo['runNumber'])]), runInfo['lumis']) fwjrSamples = ["ErrorCodeFail.json", "FailedByAgent.json", "HarvestSuccessFwjr.json", "LogCollectFailedFwjr.json", "LogCollectSuccessFwjr.json", "MergeFailedFwjr.json", "MergeSuccessFwjr.json", "NoJobReportFail.json", "ProcessingFailedFwjr.json", "ProcessingPerformanceFailed.json", "ProcessingSuccessFwjr.json", "ProductionFailedFwjr.json", "ProductionSuccessFwjr.json", "SkimSuccessFwjr.json"] for sample in fwjrSamples: sPath = os.path.join(getTestBase(), "WMCore_t/Services_t/WMArchive_t/FWJRSamples/%s" % sample) with open(sPath, 'r') as infile: fwjr = json.load(infile) job = {} job["id"] = fwjr["_id"] job['doc'] = {"fwjr": fwjr["fwjr"], "jobtype": fwjr["jobtype"], "jobstate": fwjr['jobstate'], "timestamp": fwjr["timestamp"]} newData =createArchiverDoc(job) print("\n\n==========\n%s" % sPath) pprint(newData)