def testSetLocation(self):
        """
        _testSetLocation_

        Create a file and add a couple locations.  Load the file from the
        database to make sure that the locations were set correctly.
        """
        testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10)
        testFileA.setAlgorithm(appName="cmsRun",
                               appVer="CMSSW_2_1_8",
                               appFam="RECO",
                               psetHash="GIBBERISH",
                               configContent="MOREGIBBERISH")
        testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileA.addRun(Run(1, *[45]))
        testFileA.create()

        testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"],
                              immediateSave=False)

        testFileB = DBSBufferFile(id=testFileA["id"])
        testFileB.load()

        goldenLocations = ["se1.fnal.gov", "se1.cern.ch"]

        for location in testFileB["locations"]:
            assert location in goldenLocations, \
                "ERROR: Unknown file location"
            goldenLocations.remove(location)

        assert len(goldenLocations) == 0, \
            "ERROR: Some locations are missing"
        return
Exemple #2
0
    def getFiles(self, name, tier, nFiles=12, site="malpaquet"):
        """
        Create some quick dummy test files


        """

        files = []

        for f in range(0, nFiles):
            testFile = DBSBufferFile(lfn="%s-%s-%i" % (name, site, f), size=1024, events=20, checksums={"cksum": 1})
            testFile.setAlgorithm(
                appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL
            )
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            testFile.addRun(Run(1, *[f]))
            testFile.setGlobalTag("aGlobalTag")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        testFileChild = DBSBufferFile(lfn="%s-%s-child" % (name, site), size=1024, events=10, checksums={"cksum": 1})
        testFileChild.setAlgorithm(
            appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL
        )
        testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
        testFileChild.addRun(Run(1, *[45]))
        testFileChild.setGlobalTag("aGlobalTag")
        testFileChild.create()
        testFileChild.setLocation(site)

        testFileChild.addParents([x["lfn"] for x in files])

        return files
Exemple #3
0
    def testSetLocation(self):
        """
        _testSetLocation_

        Create a file and add a couple locations.  Load the file from the
        database to make sure that the locations were set correctly.
        """
        testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10)
        testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
                               appFam = "RECO", psetHash = "GIBBERISH",
                               configContent = "MOREGIBBERISH")
        testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")        
        testFileA.addRun(Run( 1, *[45]))
        testFileA.create()
        
        testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"],
                              immediateSave = False)

        testFileB = DBSBufferFile(id = testFileA["id"])
        testFileB.load()

        goldenLocations = ["se1.fnal.gov", "se1.cern.ch"]

        for location in testFileB["locations"]:
            assert location in goldenLocations, \
                   "ERROR: Unknown file location"
            goldenLocations.remove(location)

        assert len(goldenLocations) == 0, \
              "ERROR: Some locations are missing"    
        return
Exemple #4
0
    def getFiles(self,
                 name,
                 tier='RECO',
                 nFiles=12,
                 site="malpaquet",
                 nLumis=1):
        """
        Create some quick dummy test files

        """

        files = []

        for f in range(nFiles):
            testFile = DBSBufferFile(
                lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' %
                (name, site, f),
                size=1024,
                events=20,
                checksums={'cksum': 1})
            testFile.setAlgorithm(appName=name,
                                  appVer="CMSSW_3_1_1",
                                  appFam="RECO",
                                  psetHash="GIBBERISH",
                                  configContent="MOREGIBBERISH")
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            lumis = []
            for i in range(nLumis):
                lumis.append((f * 100000) + i)
            testFile.addRun(Run(1, *lumis))
            testFile.setAcquisitionEra(name.split('-')[0])
            testFile.setProcessingVer("0")
            testFile.setGlobalTag("Weird")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        count = 0
        for f in files:
            count += 1
            testFileChild = DBSBufferFile(
                lfn=
                '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %
                (name, site, count),
                size=1024,
                events=10,
                checksums={'cksum': 1})
            testFileChild.setAlgorithm(appName=name,
                                       appVer="CMSSW_3_1_1",
                                       appFam="RECO",
                                       psetHash="GIBBERISH",
                                       configContent="MOREGIBBERISH")
            testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
            testFileChild.addRun(Run(1, *[45]))
            testFileChild.create()
            testFileChild.setLocation(site)

            testFileChild.addParents([f['lfn']])

        return files
    def prepareDBSFiles(self):
        """
        _prepareDBSFiles_

        Retrieve the information from the JSON input data
        and create DBSFile objects that can be registered in the
        database.
        """
        timestamp = time.strftime('%m%d%y_%H%M%S')
        for fileEntry in self.inputData:
            # Get all the info out of a standard named dataset
            datasetInfo = str(fileEntry["dataset"])
            tokens = datasetInfo.split('/')
            primDs = tokens[1]
            procDs = tokens[2]
            dataTier = tokens[3]
            procDsTokens = procDs.split('-')
            acqEra = procDsTokens[0]
            procVer = procDsTokens[-1][1:]

            ckSumInfo = fileEntry["checksums"]
            for entry in ckSumInfo:
                ckSumInfo[entry] = str(ckSumInfo[entry])

            # Build the basic dbsBuffer file
            dbsFile = DBSBufferFile(lfn = str(fileEntry["lfn"]),
                                    size = int(fileEntry.get("size", 0)),
                                    events = int(fileEntry.get("events", 0)),
                                    checksums = ckSumInfo,
                                    status = "NOTUPLOADED")
            dbsFile.setAlgorithm(appName = "cmsRun",
                                 appVer = str(fileEntry.get("cmssw", "LEGACY")),
                                 appFam = "Legacy",
                                 psetHash = "GIBBERISH",
                                 configContent = "None;;None;;None")

            dbsFile.setDatasetPath("/%s/%s/%s" % (primDs,
                                                  procDs,
                                                  dataTier))
            dbsFile.setValidStatus(validStatus = "PRODUCTION")
            dbsFile.setProcessingVer(ver = procVer)
            dbsFile.setAcquisitionEra(era = acqEra)
            dbsFile.setGlobalTag(globalTag = str(fileEntry.get('globalTag', "LEGACY")))

            # Build a representative task name
            dbsFile['task'] = '/LegacyInsertionTask_%s/Insertion' % timestamp

            # Get the runs and lumis
            runsAndLumis = fileEntry.get("runsAndLumis", {})
            for run in runsAndLumis:
                newRun = Run(runNumber = int(run))
                newRun.extend([int(x) for x in runsAndLumis[run]])
                dbsFile.addRun(newRun)

            # Complete the file information with the location and queue it
            dbsFile.setLocation(se = str(fileEntry["location"]), immediateSave = False)
            self.dbsFilesToCreate.append(dbsFile)
        self.inputData = None
        return
Exemple #6
0
    def getFiles(self,
                 name,
                 tier,
                 nFiles=12,
                 site="malpaquet",
                 workflowName=None,
                 taskPath=None,
                 noChild=False):
        """
        Create some quick dummy test files
        """

        if workflowName is not None and taskPath is not None:
            workflowId = self.injectWorkflow(workflowName=workflowName,
                                             taskPath=taskPath)
        else:
            workflowId = self.injectWorkflow()

        files = []

        for f in range(0, nFiles):
            testFile = DBSBufferFile(lfn='%s-%s-%i' % (name, site, f),
                                     size=1024,
                                     events=20,
                                     checksums={'cksum': 1},
                                     workflowId=workflowId)
            testFile.setAlgorithm(appName=name,
                                  appVer="CMSSW_3_1_1",
                                  appFam="RECO",
                                  psetHash="GIBBERISH",
                                  configContent=self.configURL)
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            testFile.addRun(Run(1, *[f]))
            testFile.setGlobalTag("aGlobalTag")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        if not noChild:
            testFileChild = DBSBufferFile(lfn='%s-%s-child' % (name, site),
                                          size=1024,
                                          events=10,
                                          checksums={'cksum': 1},
                                          workflowId=workflowId)
            testFileChild.setAlgorithm(appName=name,
                                       appVer="CMSSW_3_1_1",
                                       appFam="RECO",
                                       psetHash="GIBBERISH",
                                       configContent=self.configURL)
            testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
            testFileChild.addRun(Run(1, *[45]))
            testFileChild.setGlobalTag("aGlobalTag")
            testFileChild.create()
            testFileChild.setLocation(site)

            testFileChild.addParents([x['lfn'] for x in files])

        return files
Exemple #7
0
    def createFilesWithChildren(self, moreParentFiles, acqEra):
        """
        _createFilesWithChildren_

        Create several parentless files and then create child files.
        """
        parentFiles = []
        childFiles = []
        
        baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra)
        for i in range(10):
            testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024,
                                     events = 20, checksums = {"cksum": 1})
            testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1",
                                  appFam = "RAW", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra))

            lumis = []
            for j in range(10):
                lumis.append((i * 10) + j)
            testFile.addRun(Run(143316, *lumis))            

            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("1")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation("malpaquet")
            parentFiles.append(testFile)

        baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % (acqEra)
        for i in range(5):
            testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024,
                                     events = 20, checksums = {"cksum": 1})
            testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % (acqEra))

            lumis = []
            for j in range(20):
                lumis.append((i * 20) + j)
            testFile.addRun(Run(143316, *lumis))            

            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("1")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation("malpaquet")
            testFile.addParents([parentFiles[i * 2]["lfn"],
                                 parentFiles[i * 2 + 1]["lfn"]])
            testFile.addParents([moreParentFiles[i * 2]["lfn"],
                                 moreParentFiles[i * 2 + 1]["lfn"]])
            childFiles.append(testFile)            

        return (parentFiles, childFiles)
Exemple #8
0
    def getFiles(self, name, tier, nFiles=12, site="malpaquet", nLumis=1):
        """
        _getFiles_
        
        Create some dummy test files.
        """
        files = []

        (acqEra, procVer) = name.split("-")
        baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra,
                                                                   procVer)
        for f in range(nFiles):
            testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root",
                                     size=1024,
                                     events=20,
                                     checksums={"cksum": 1})
            testFile.setAlgorithm(appName="cmsRun",
                                  appVer="CMSSW_3_1_1",
                                  appFam="RECO",
                                  psetHash="GIBBERISH",
                                  configContent="MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer))
            lumis = []
            for i in range(nLumis):
                lumis.append((f * 1000000) + i)
            testFile.addRun(Run(1, *lumis))
            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("0")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra,
                                                                       procVer)
        testFileChild = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root",
                                      size=1024,
                                      events=10,
                                      checksums={'cksum': 1})
        testFileChild.setAlgorithm(appName="cmsRun",
                                   appVer="CMSSW_3_1_1",
                                   appFam="RAW-RECO",
                                   psetHash="GIBBERISH",
                                   configContent="MOREGIBBERISH")
        testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" %
                                     (acqEra, procVer))
        testFileChild.addRun(Run(1, *[45]))
        testFileChild.create()
        testFileChild.setLocation(site)

        testFileChild.addParents([x['lfn'] for x in files])
        return files
Exemple #9
0
    def addFileToDBS(self, jobReportFile, task, errorDataset=False):
        """
        _addFileToDBS_

        Add a file that was output from a job to the DBS buffer.
        """
        datasetInfo = jobReportFile["dataset"]

        dbsFile = DBSBufferFile(lfn=jobReportFile["lfn"],
                                size=jobReportFile["size"],
                                events=jobReportFile["events"],
                                checksums=jobReportFile["checksums"],
                                status="NOTUPLOADED")
        dbsFile.setAlgorithm(appName=datasetInfo["applicationName"],
                             appVer=datasetInfo["applicationVersion"],
                             appFam=jobReportFile["module_label"],
                             psetHash="GIBBERISH",
                             configContent=jobReportFile.get('configURL'))

        if errorDataset:
            dbsFile.setDatasetPath(
                "/%s/%s/%s" %
                (datasetInfo["primaryDataset"] + "-Error",
                 datasetInfo["processedDataset"], datasetInfo["dataTier"]))
        else:
            dbsFile.setDatasetPath(
                "/%s/%s/%s" %
                (datasetInfo["primaryDataset"],
                 datasetInfo["processedDataset"], datasetInfo["dataTier"]))

        dbsFile.setValidStatus(
            validStatus=jobReportFile.get("validStatus", None))
        dbsFile.setProcessingVer(ver=jobReportFile.get('processingVer', None))
        dbsFile.setAcquisitionEra(
            era=jobReportFile.get('acquisitionEra', None))
        dbsFile.setGlobalTag(globalTag=jobReportFile.get('globalTag', None))
        #TODO need to find where to get the prep id
        dbsFile.setPrepID(prep_id=jobReportFile.get('prep_id', None))
        dbsFile['task'] = task

        for run in jobReportFile["runs"]:
            newRun = Run(runNumber=run.run)
            newRun.extend(run.lumis)
            dbsFile.addRun(newRun)

        dbsFile.setLocation(pnn=list(jobReportFile["locations"])[0],
                            immediateSave=False)
        self.dbsFilesToCreate.append(dbsFile)
        return
Exemple #10
0
    def createParentFiles(self,
                          acqEra,
                          nFiles=10,
                          workflowName='TestWorkload',
                          taskPath='/TestWorkload/DataTest'):
        """
        _createParentFiles_

        Create several parentless files in DBSBuffer.  This simulates raw files
        in the T0.
        """
        workflowId = self.injectWorkflow(workflowName=workflowName,
                                         taskPath=taskPath)
        parentlessFiles = []

        baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra)
        for i in range(nFiles):
            testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root",
                                     size=1024,
                                     events=20,
                                     checksums={"cksum": 1},
                                     workflowId=workflowId)
            testFile.setAlgorithm(appName="cmsRun",
                                  appVer="CMSSW_3_1_1",
                                  appFam="RAW",
                                  psetHash="GIBBERISH",
                                  configContent="MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra))

            testFile['block_close_max_wait_time'] = 1000000
            testFile['block_close_max_events'] = 1000000
            testFile['block_close_max_size'] = 1000000
            testFile['block_close_max_files'] = 1000000

            lumis = []
            for j in range(10):
                lumis.append((i * 10) + j)
            testFile.addRun(Run(143316, *lumis))

            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("1")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation("malpaquet")
            parentlessFiles.append(testFile)

        return parentlessFiles
Exemple #11
0
    def addFileToDBS(self, jobReportFile, task, errorDataset = False):
        """
        _addFileToDBS_

        Add a file that was output from a job to the DBS buffer.
        """
        datasetInfo = jobReportFile["dataset"]

        dbsFile = DBSBufferFile(lfn = jobReportFile["lfn"],
                                size = jobReportFile["size"],
                                events = jobReportFile["events"],
                                checksums = jobReportFile["checksums"],
                                status = "NOTUPLOADED")
        dbsFile.setAlgorithm(appName = datasetInfo["applicationName"],
                             appVer = datasetInfo["applicationVersion"],
                             appFam = jobReportFile["module_label"],
                             psetHash = "GIBBERISH",
                             configContent = jobReportFile.get('configURL'))

        if errorDataset:
            dbsFile.setDatasetPath("/%s/%s/%s" % (datasetInfo["primaryDataset"] + "-Error",
                                                  datasetInfo["processedDataset"],
                                                  datasetInfo["dataTier"]))
        else:
            dbsFile.setDatasetPath("/%s/%s/%s" % (datasetInfo["primaryDataset"],
                                                  datasetInfo["processedDataset"],
                                                  datasetInfo["dataTier"]))

        dbsFile.setValidStatus(validStatus = jobReportFile.get("validStatus", None))
        dbsFile.setProcessingVer(ver = jobReportFile.get('processingVer', None))
        dbsFile.setAcquisitionEra(era = jobReportFile.get('acquisitionEra', None))
        dbsFile.setGlobalTag(globalTag = jobReportFile.get('globalTag', None))
        #TODO need to find where to get the prep id
        dbsFile.setPrepID(prep_id = jobReportFile.get('prep_id', None))
        dbsFile['task'] = task

        for run in jobReportFile["runs"]:
            newRun = Run(runNumber = run.run)
            newRun.extend(run.lumis)
            dbsFile.addRun(newRun)


        dbsFile.setLocation(pnn = list(jobReportFile["locations"])[0], immediateSave = False)
        self.dbsFilesToCreate.append(dbsFile)
        return
    def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", workflowName = None, taskPath = None,
                 noChild = False):
        """
        Create some quick dummy test files
        """

        if workflowName is not None and taskPath is not None:
            workflowId = self.injectWorkflow(workflowName = workflowName,
                                             taskPath = taskPath)
        else:
            workflowId = self.injectWorkflow()

        files = []

        for f in range(0, nFiles):
            testFile = DBSBufferFile(lfn = '%s-%s-%i' % (name, site, f), size = 1024,
                                     events = 20, checksums = {'cksum': 1}, workflowId = workflowId)
            testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = self.configURL)
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            testFile.addRun(Run( 1, *[f]))
            testFile.setGlobalTag("aGlobalTag")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        if not noChild:
            testFileChild = DBSBufferFile(lfn = '%s-%s-child' %(name, site), size = 1024,
                                     events = 10, checksums = {'cksum': 1},
                                     workflowId = workflowId)
            testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = self.configURL)
            testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name))
            testFileChild.addRun(Run( 1, *[45]))
            testFileChild.setGlobalTag("aGlobalTag")
            testFileChild.create()
            testFileChild.setLocation(site)

            testFileChild.addParents([x['lfn'] for x in files])


        return files
Exemple #13
0
    def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1):
        """
        Create some quick dummy test files

        """

        files = []

        for f in range(nFiles):
            testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024,
                                     events = 20, checksums = {'cksum': 1})
            testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            lumis = []
            for i in range(nLumis):
                lumis.append((f * 100000) + i)
            testFile.addRun(Run( 1, *lumis))
            testFile.setAcquisitionEra(name.split('-')[0])
            testFile.setProcessingVer("0")
            testFile.setGlobalTag("Weird")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        count = 0
        for f in files:
            count += 1
            testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024,
                                          events = 10, checksums = {'cksum': 1})
            testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
                                       appFam = "RECO", psetHash = "GIBBERISH",
                                       configContent = "MOREGIBBERISH")
            testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name))
            testFileChild.addRun(Run( 1, *[45]))
            testFileChild.create()
            testFileChild.setLocation(site)

            testFileChild.addParents([f['lfn']])


        return files
Exemple #14
0
    def addFile(self, file, dataset=0):
        """
        Add the file to the buffer
        """

        myThread = threading.currentThread()

        existingTransaction = self.beginTransaction()

        bufferFile = DBSBufferFile(lfn=file['LFN'],
                                   size=file['Size'],
                                   events=file['TotalEvents'],
                                   cksum=file['Checksum'],
                                   dataset=dataset)

        runLumiList = file.getLumiSections()
        runList = [x['RunNumber'] for x in runLumiList]

        for runNumber in runList:
            lumis = [
                int(y['LumiSectionNumber']) for y in runLumiList
                if y['RunNumber'] == runNumber
            ]
            run = Run(runNumber, *lumis)
            bufferFile.addRun(run)

        if bufferFile.exists() == False:
            bufferFile.create()
            bufferFile.setLocation(se=file['SEName'], immediateSave=True)
        else:
            bufferFile.load()
        # Lets add the file to DBS Buffer as well
        #UPDATE File Count

        self.updateDSFileCount(dataset=dataset)

        #Parent files
        bufferFile.addParents(file.inputFiles)

        self.commitTransaction(existingTransaction)

        return
Exemple #15
0
    def createParentFiles(self, acqEra, nFiles=10, workflowName="TestWorkload", taskPath="/TestWorkload/DataTest"):
        """
        _createParentFiles_

        Create several parentless files in DBSBuffer.  This simulates raw files
        in the T0.
        """
        workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath)
        parentlessFiles = []

        baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra)
        for i in range(nFiles):
            testFile = DBSBufferFile(
                lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}, workflowId=workflowId
            )
            testFile.setAlgorithm(
                appName="cmsRun",
                appVer="CMSSW_3_1_1",
                appFam="RAW",
                psetHash="GIBBERISH",
                configContent="MOREGIBBERISH",
            )
            testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra))

            testFile["block_close_max_wait_time"] = 1000000
            testFile["block_close_max_events"] = 1000000
            testFile["block_close_max_size"] = 1000000
            testFile["block_close_max_files"] = 1000000

            lumis = []
            for j in range(10):
                lumis.append((i * 10) + j)
            testFile.addRun(Run(143316, *lumis))

            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("1")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation("malpaquet")
            parentlessFiles.append(testFile)

        return parentlessFiles
Exemple #16
0
    def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", nLumis = 1):
        """
        _getFiles_
        
        Create some dummy test files.
        """
        files = []

        (acqEra, procVer) = name.split("-")
        baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra, procVer)
        for f in range(nFiles):
            testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024,
                                     events = 20, checksums = {"cksum": 1})
            testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer))
            lumis = []
            for i in range(nLumis):
                lumis.append((f * 1000000) + i)
            testFile.addRun(Run( 1, *lumis))
            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("0")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra, procVer)
        testFileChild = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root",
                                      size = 1024, events = 10, checksums = {'cksum': 1})
        testFileChild.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1",
                              appFam = "RAW-RECO", psetHash = "GIBBERISH",
                              configContent = "MOREGIBBERISH")
        testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" %(acqEra, procVer))
        testFileChild.addRun(Run( 1, *[45]))
        testFileChild.create()
        testFileChild.setLocation(site)

        testFileChild.addParents([x['lfn'] for x in files])
        return files
Exemple #17
0
    def addFile(self, file, dataset=0):
        """
        Add the file to the buffer
        """

        myThread = threading.currentThread()

        existingTransaction = self.beginTransaction()

        bufferFile = DBSBufferFile(
            lfn=file["LFN"], size=file["Size"], events=file["TotalEvents"], cksum=file["Checksum"], dataset=dataset
        )

        runLumiList = file.getLumiSections()
        runList = [x["RunNumber"] for x in runLumiList]

        for runNumber in runList:
            lumis = [int(y["LumiSectionNumber"]) for y in runLumiList if y["RunNumber"] == runNumber]
            run = Run(runNumber, *lumis)
            bufferFile.addRun(run)

        if bufferFile.exists() == False:
            bufferFile.create()
            bufferFile.setLocation(se=file["SEName"], immediateSave=True)
        else:
            bufferFile.load()
        # Lets add the file to DBS Buffer as well
        # UPDATE File Count

        self.updateDSFileCount(dataset=dataset)

        # Parent files
        bufferFile.addParents(file.inputFiles)

        self.commitTransaction(existingTransaction)

        return
Exemple #18
0
    def testSetLocationTransaction(self):
        """
        _testSetLocationTransaction_

        Create a file at specific locations and commit everything to the
        database.  Reload the file from the database and verify that the
        locations are correct.  Rollback the database transaction and once
        again reload the file.  Verify that the original locations are back.
        """
        testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10)
        testFileA.setAlgorithm(appName="cmsRun",
                               appVer="CMSSW_2_1_8",
                               appFam="RECO",
                               psetHash="GIBBERISH",
                               configContent="MOREGIBBERISH")
        testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileA.addRun(Run(1, *[45]))
        testFileA.create()

        testFileA.setLocation(["se1.fnal.gov"])

        myThread = threading.currentThread()
        myThread.transaction.begin()

        testFileA.setLocation(["se1.cern.ch"])
        testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"],
                              immediateSave=False)

        testFileB = DBSBufferFile(id=testFileA["id"])
        testFileB.load()

        goldenLocations = ["se1.fnal.gov", "se1.cern.ch"]

        for location in testFileB["locations"]:
            assert location in goldenLocations, \
                "ERROR: Unknown file location"
            goldenLocations.remove(location)

        assert len(goldenLocations) == 0, \
            "ERROR: Some locations are missing"

        myThread.transaction.rollback()
        testFileB.load()

        goldenLocations = ["se1.fnal.gov"]

        for location in testFileB["locations"]:
            assert location in goldenLocations, \
                "ERROR: Unknown file location"
            goldenLocations.remove(location)

        assert len(goldenLocations) == 0, \
            "ERROR: Some locations are missing"
        return
Exemple #19
0
    def testSetLocationTransaction(self):
        """
        _testSetLocationTransaction_

        Create a file at specific locations and commit everything to the
        database.  Reload the file from the database and verify that the
        locations are correct.  Rollback the database transaction and once
        again reload the file.  Verify that the original locations are back.
        """
        testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10)
        testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
                               appFam = "RECO", psetHash = "GIBBERISH",
                               configContent = "MOREGIBBERISH")
        testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")        
        testFileA.addRun(Run( 1, *[45]))
        testFileA.create()
        
        testFileA.setLocation(["se1.fnal.gov"])

        myThread = threading.currentThread()
        myThread.transaction.begin()
        
        testFileA.setLocation(["se1.cern.ch"])
        testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"],
                              immediateSave = False)

        testFileB = DBSBufferFile(id = testFileA["id"])
        testFileB.load()

        goldenLocations = ["se1.fnal.gov", "se1.cern.ch"]

        for location in testFileB["locations"]:
            assert location in goldenLocations, \
                   "ERROR: Unknown file location"
            goldenLocations.remove(location)

        assert len(goldenLocations) == 0, \
              "ERROR: Some locations are missing"

        myThread.transaction.rollback()
        testFileB.load()

        goldenLocations = ["se1.fnal.gov"]

        for location in testFileB["locations"]:
            assert location in goldenLocations, \
                   "ERROR: Unknown file location"
            goldenLocations.remove(location)

        assert len(goldenLocations) == 0, \
              "ERROR: Some locations are missing"
        return    
Exemple #20
0
    def testBulkLoad(self):
        """
        _testBulkLoad_

        Can we load in bulk?
        """


        addToBuffer = DBSBufferUtil()

        bulkLoad = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID")


        testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024,
                                        events = 20)
        testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
                                    appFam = "RECO", psetHash = "GIBBERISH",
                                    configContent = "MOREGIBBERISH")
        testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024,
                                        events = 20)
        testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
                                    appFam = "RECO", psetHash = "GIBBERISH",
                                    configContent = "MOREGIBBERISH")
        testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")        
        testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024,
                                        events = 20)
        testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
                                    appFam = "RECO", psetHash = "GIBBERISH",
                                    configContent = "MOREGIBBERISH")
        testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")        
        
        testFileChildA.create()
        testFileChildB.create()
        testFileChildC.create()

        testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"])

        runSet = set()
        runSet.add(Run( 1, *[45]))
        runSet.add(Run( 2, *[67, 68]))
        testFileChildA.addRunSet(runSet)
        testFileChildB.addRunSet(runSet)
        testFileChildC.addRunSet(runSet)


        testFileChildA.save()
        testFileChildB.save()
        testFileChildC.save()

        setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
        binds = [{'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201},
                 {'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101},
                 {'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201},
                 {'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101},
                 {'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201},
                 {'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101}]
        setCksumAction.execute(bulkList = binds)        

        testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024,
                                 events = 10)
        testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
                              appFam = "RECO", psetHash = "GIBBERISH",
                              configContent = "MOREGIBBERISH")
        testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")        
        testFile.create()

        testFileChildA.addParents([testFile["lfn"]])
        testFileChildB.addParents([testFile["lfn"]])
        testFileChildC.addParents([testFile["lfn"]])


        binds = [{'id': testFileChildA.exists()},
                 {'id': testFileChildB.exists()},
                 {'id': testFileChildC.exists()}]

        listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs = binds)

        #print listOfFiles


        compareList = ['locations', 'psetHash', 'configContent', 'appName',
                       'appVer', 'appFam', 'events', 'datasetPath', 'runs']


        for f in listOfFiles:
            self.assertTrue(f['lfn'] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB",
                                         "/this/is/a/child/lfnC"],
                            "Unknown file in loaded results")
            self.assertEqual(f['checksums'], {'adler32': '201', 'cksum': '101'})
            for parent in f['parents']:
                self.assertEqual(parent['lfn'], testFile['lfn'])
            for key in compareList:
                self.assertEqual(f[key], testFileChildA[key])
Exemple #21
0
    def testBulkLoad(self):
        """
        _testBulkLoad_

        Can we load in bulk?
        """

        addToBuffer = DBSBufferUtil()

        testFileChildA = DBSBufferFile(lfn="/this/is/a/child/lfnA",
                                       size=1024,
                                       events=20)
        testFileChildA.setAlgorithm(appName="cmsRun",
                                    appVer="CMSSW_2_1_8",
                                    appFam="RECO",
                                    psetHash="GIBBERISH",
                                    configContent="MOREGIBBERISH")
        testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileChildB = DBSBufferFile(lfn="/this/is/a/child/lfnB",
                                       size=1024,
                                       events=20)
        testFileChildB.setAlgorithm(appName="cmsRun",
                                    appVer="CMSSW_2_1_8",
                                    appFam="RECO",
                                    psetHash="GIBBERISH",
                                    configContent="MOREGIBBERISH")
        testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileChildC = DBSBufferFile(lfn="/this/is/a/child/lfnC",
                                       size=1024,
                                       events=20)
        testFileChildC.setAlgorithm(appName="cmsRun",
                                    appVer="CMSSW_2_1_8",
                                    appFam="RECO",
                                    psetHash="GIBBERISH",
                                    configContent="MOREGIBBERISH")
        testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")

        testFileChildA.create()
        testFileChildB.create()
        testFileChildC.create()

        testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"])

        runSet = set()
        runSet.add(Run(1, *[45]))
        runSet.add(Run(2, *[67, 68]))
        testFileChildA.addRunSet(runSet)
        testFileChildB.addRunSet(runSet)
        testFileChildC.addRunSet(runSet)

        testFileChildA.save()
        testFileChildB.save()
        testFileChildC.save()

        setCksumAction = self.daoFactory(
            classname="DBSBufferFiles.AddChecksumByLFN")
        binds = [{
            'lfn': "/this/is/a/child/lfnA",
            'cktype': 'adler32',
            'cksum': 201
        }, {
            'lfn': "/this/is/a/child/lfnA",
            'cktype': 'cksum',
            'cksum': 101
        }, {
            'lfn': "/this/is/a/child/lfnB",
            'cktype': 'adler32',
            'cksum': 201
        }, {
            'lfn': "/this/is/a/child/lfnB",
            'cktype': 'cksum',
            'cksum': 101
        }, {
            'lfn': "/this/is/a/child/lfnC",
            'cktype': 'adler32',
            'cksum': 201
        }, {
            'lfn': "/this/is/a/child/lfnC",
            'cktype': 'cksum',
            'cksum': 101
        }]
        setCksumAction.execute(bulkList=binds)

        testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10)
        testFile.setAlgorithm(appName="cmsRun",
                              appVer="CMSSW_2_1_8",
                              appFam="RECO",
                              psetHash="GIBBERISH",
                              configContent="MOREGIBBERISH")
        testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFile.create()

        testFileChildA.addParents([testFile["lfn"]])
        testFileChildB.addParents([testFile["lfn"]])
        testFileChildC.addParents([testFile["lfn"]])

        binds = [{
            'id': testFileChildA.exists()
        }, {
            'id': testFileChildB.exists()
        }, {
            'id': testFileChildC.exists()
        }]

        listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs=binds)

        # print listOfFiles

        compareList = [
            'locations', 'psetHash', 'configContent', 'appName', 'appVer',
            'appFam', 'events', 'datasetPath', 'runs'
        ]

        for f in listOfFiles:
            self.assertTrue(
                f['lfn'] in [
                    "/this/is/a/child/lfnA", "/this/is/a/child/lfnB",
                    "/this/is/a/child/lfnC"
                ], "Unknown file in loaded results")
            self.assertEqual(f['checksums'], {
                'adler32': '201',
                'cksum': '101'
            })
            for parent in f['parents']:
                self.assertEqual(parent['lfn'], testFile['lfn'])
            for key in compareList:
                self.assertEqual(f[key], testFileChildA[key])
Exemple #22
0
    def createFilesWithChildren(self, moreParentFiles, acqEra):
        """
        _createFilesWithChildren_

        Create several parentless files and then create child files.
        """
        parentFiles = []
        childFiles = []

        baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra)
        for i in range(10):
            testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024,
                                     events = 20, checksums = {"cksum": 1})
            testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1",
                                  appFam = "RAW", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra))

            testFile['block_close_max_wait_time'] = 1000000
            testFile['block_close_max_events'] = 1000000
            testFile['block_close_max_size'] = 1000000
            testFile['block_close_max_files'] = 1000000

            lumis = []
            for j in range(10):
                lumis.append((i * 10) + j)
            testFile.addRun(Run(143316, *lumis))

            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("1")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation("malpaquet")
            parentFiles.append(testFile)

        baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % (acqEra)
        for i in range(5):
            testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024,
                                     events = 20, checksums = {"cksum": 1})
            testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % (acqEra))

            testFile['block_close_max_wait_time'] = 1000000
            testFile['block_close_max_events'] = 1000000
            testFile['block_close_max_size'] = 1000000
            testFile['block_close_max_files'] = 1000000

            lumis = []
            for j in range(20):
                lumis.append((i * 20) + j)
            testFile.addRun(Run(143316, *lumis))

            testFile.setAcquisitionEra(acqEra)
            testFile.setProcessingVer("1")
            testFile.setGlobalTag("START54::All")
            testFile.create()
            testFile.setLocation("malpaquet")
            testFile.addParents([parentFiles[i * 2]["lfn"],
                                 parentFiles[i * 2 + 1]["lfn"]])
            testFile.addParents([moreParentFiles[i * 2]["lfn"],
                                 moreParentFiles[i * 2 + 1]["lfn"]])
            childFiles.append(testFile)

        return (parentFiles, childFiles)
Exemple #23
0
    def testBulkLoad(self):
        """
        _testBulkLoad_

        Can we load in bulk?
        """

        bufferFactory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database.Interface")
        addToBuffer = bufferFactory.loadObject("AddToBuffer")

        bulkLoad = self.daoFactory(classname="DBSBufferFiles.LoadBulkFilesByID")

        testFileChildA = DBSBufferFile(lfn="/this/is/a/child/lfnA", size=1024, events=20)
        testFileChildA.setAlgorithm(
            appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
        )
        testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileChildB = DBSBufferFile(lfn="/this/is/a/child/lfnB", size=1024, events=20)
        testFileChildB.setAlgorithm(
            appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
        )
        testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFileChildC = DBSBufferFile(lfn="/this/is/a/child/lfnC", size=1024, events=20)
        testFileChildC.setAlgorithm(
            appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
        )
        testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")

        testFileChildA.create()
        testFileChildB.create()
        testFileChildC.create()

        testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"])
        testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"])

        runSet = set()
        runSet.add(Run(1, *[45]))
        runSet.add(Run(2, *[67, 68]))
        testFileChildA.addRunSet(runSet)
        testFileChildB.addRunSet(runSet)
        testFileChildC.addRunSet(runSet)

        testFileChildA.save()
        testFileChildB.save()
        testFileChildC.save()

        setCksumAction = self.daoFactory(classname="DBSBufferFiles.AddChecksumByLFN")
        binds = [
            {"lfn": "/this/is/a/child/lfnA", "cktype": "adler32", "cksum": 201},
            {"lfn": "/this/is/a/child/lfnA", "cktype": "cksum", "cksum": 101},
            {"lfn": "/this/is/a/child/lfnB", "cktype": "adler32", "cksum": 201},
            {"lfn": "/this/is/a/child/lfnB", "cktype": "cksum", "cksum": 101},
            {"lfn": "/this/is/a/child/lfnC", "cktype": "adler32", "cksum": 201},
            {"lfn": "/this/is/a/child/lfnC", "cktype": "cksum", "cksum": 101},
        ]
        setCksumAction.execute(bulkList=binds)

        testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10)
        testFile.setAlgorithm(
            appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
        )
        testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
        testFile.create()

        testFileChildA.addParents([testFile["lfn"]])
        testFileChildB.addParents([testFile["lfn"]])
        testFileChildC.addParents([testFile["lfn"]])

        binds = [{"id": testFileChildA.exists()}, {"id": testFileChildB.exists()}, {"id": testFileChildC.exists()}]

        listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs=binds)

        # print listOfFiles

        compareList = [
            "locations",
            "psetHash",
            "configContent",
            "appName",
            "appVer",
            "appFam",
            "events",
            "datasetPath",
            "runs",
        ]

        for f in listOfFiles:
            self.assertTrue(
                f["lfn"] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC"],
                "Unknown file in loaded results",
            )
            self.assertEqual(f["checksums"], {"adler32": "201", "cksum": "101"})
            for parent in f["parents"]:
                self.assertEqual(parent["lfn"], testFile["lfn"])
            for key in compareList:
                self.assertEqual(f[key], testFileChildA[key])