Exemple #1
0
    def addInputQuery(self, inDS):
        """Add the given input dataset to the list but only if BK queries aren't given"""
        if len(self.queries) > 0:
            logger.error(
                "Cannot add both input data and BK queries. Query already given")
            return

        super(LHCbTransform, self).addInputQuery(inDS)
Exemple #2
0
    def addInputQuery(self, inDS):
        """Add the given input dataset to the list but only if BK queries aren't given"""
        if len(self.queries) > 0:
            logger.error(
                "Cannot add both input data and BK queries. Query already given"
            )
            return

        super(LHCbTransform, self).addInputQuery(inDS)
Exemple #3
0
    def addQuery(self, bk):
        """Add a BK query to this transform"""
        # Check if the BKQuery input is correct and append/update
        if not isType(bk, BKQuery):
            raise GangaAttributeError(
                None, 'LHCbTransform expects a BKQuery object passed to the addQuery method')

        # check we don't already have inputdata
        if len(self.queries) == 0 and len(self.inputdata) > 0:
            logger.error(
                "Cannot add both input data and BK queries. Input Data already present.")
            return

        # add the query and update the input data
        self.queries.append(bk)
        self.updateQuery()
Exemple #4
0
    def addQuery(self, bk):
        """Add a BK query to this transform"""
        # Check if the BKQuery input is correct and append/update
        if not isType(bk, BKQuery):
            raise GangaAttributeError(
                None,
                'LHCbTransform expects a BKQuery object passed to the addQuery method'
            )

        # check we don't already have inputdata
        if len(self.queries) == 0 and len(self.inputdata) > 0:
            logger.error(
                "Cannot add both input data and BK queries. Input Data already present."
            )
            return

        # add the query and update the input data
        self.queries.append(bk)
        self.updateQuery()
Exemple #5
0
    def removeUnusedData(self):
        """Remove any output data from orphaned jobs"""
        for unit in self.units:
            for jid in unit.prev_job_ids:
                try:
                    logger.warning("Removing data from job '%d'..." % jid)
                    job = GPI.jobs(jid)

                    jlist = []
                    if len(job.subjobs) > 0:
                        jlist = job.subjobs
                    else:
                        jlist = [job]

                    for sj in jlist:
                        for f in sj.outputfiles:
                            if isType(f, DiracFile) == "DiracFile" and f.lfn:
                                f.remove()
                except:
                    logger.error("Problem deleting data for job '%d'" % jid)
                    pass
Exemple #6
0
    def removeUnusedData(self):
        """Remove any output data from orphaned jobs"""
        for unit in self.units:
            for jid in unit.prev_job_ids:
                try:
                    logger.warning("Removing data from job '%d'..." % jid)
                    job = GPI.jobs(jid)

                    jlist = []
                    if len(job.subjobs) > 0:
                        jlist = job.subjobs
                    else:
                        jlist = [job]

                    for sj in jlist:
                        for f in sj.outputfiles:
                            if isType(f, DiracFile) == "DiracFile" and f.lfn:
                                f.remove()
                except:
                    logger.error("Problem deleting data for job '%d'" % jid)
                    pass
Exemple #7
0
    def createUnits(self):
        """Create new units if required given the inputdata"""

        # call parent for chaining
        super(LHCbTransform, self).createUnits()

        if len(self.inputdata) > 0:

            # check for conflicting input
            if self.mc_num_units > 0:
                logger.warning("Inputdata specified - MC Event info ignored")

            # loop over input data and see if we need to create any more units
            import copy
            for id, inds in enumerate(self.inputdata):

                if not isType(inds, LHCbDataset):
                    continue

                # go over the units and see what files have been assigned
                assigned_data = LHCbDataset()
                for unit in self.units:

                    if unit.input_datset_index != id:
                        continue

                    assigned_data.files += unit.inputdata.files

                # any new files
                new_data = LHCbDataset( files = self.inputdata[id].difference(assigned_data).files )

                if len(new_data.files) == 0:
                    continue

                # create units for these files
                if self.files_per_unit > 0:

                    # loop over the file array and create units for each set
                    num = 0
                    while num < len(new_data.files):
                        unit = LHCbUnit()
                        unit.name = "Unit %d" % len(self.units)
                        unit.input_datset_index = id
                        self.addUnitToTRF(unit)
                        unit.inputdata = copy.deepcopy(self.inputdata[id])
                        unit.inputdata.files = []
                        unit.inputdata.files += new_data.files[
                            num:num + self.files_per_unit]
                        num += self.files_per_unit

                else:
                    # new unit required for this dataset
                    unit = LHCbUnit()
                    unit.name = "Unit %d" % len(self.units)
                    self.addUnitToTRF(unit)
                    unit.inputdata = copy.deepcopy(self.inputdata[id])
                    unit.inputdata.files = []
                    unit.inputdata.files += new_data.files

        elif self.mc_num_units > 0:

            if len(self.units) == 0:
                # check for appropriate splitter
                from GPI import GaussSplitter
                if not self.splitter or isType(self.splitter, GaussSplitter):
                    logger.warning("No GaussSplitter specified - first event info ignored")

                # create units for MC generation
                for i in range(0, self.mc_num_units):
                    unit = LHCbUnit()
                    unit.name = "Unit %d" % len(self.units)
                    self.addUnitToTRF(unit)
        else:
            logger.error(
                "Please specify either inputdata or MC info for unit generation")
Exemple #8
0
    def createUnits(self):
        """Create new units if required given the inputdata"""

        # call parent for chaining
        super(LHCbTransform, self).createUnits()

        if len(self.inputdata) > 0:

            # check for conflicting input
            if self.mc_num_units > 0:
                logger.warning("Inputdata specified - MC Event info ignored")

            # loop over input data and see if we need to create any more units
            import copy
            for id, inds in enumerate(self.inputdata):

                if not isType(inds, LHCbDataset):
                    continue

                # go over the units and see what files have been assigned
                assigned_data = LHCbDataset()
                for unit in self.units:

                    if unit.input_datset_index != id:
                        continue

                    assigned_data.files += unit.inputdata.files

                # any new files
                new_data = LHCbDataset(
                    files=self.inputdata[id].difference(assigned_data).files)

                if len(new_data.files) == 0:
                    continue

                # create units for these files
                if self.files_per_unit > 0:

                    # loop over the file array and create units for each set
                    num = 0
                    while num < len(new_data.files):
                        unit = LHCbUnit()
                        unit.name = "Unit %d" % len(self.units)
                        unit.input_datset_index = id
                        self.addUnitToTRF(unit)
                        unit.inputdata = copy.deepcopy(self.inputdata[id])
                        unit.inputdata.files = []
                        unit.inputdata.files += new_data.files[num:num + self.
                                                               files_per_unit]
                        num += self.files_per_unit

                else:
                    # new unit required for this dataset
                    unit = LHCbUnit()
                    unit.name = "Unit %d" % len(self.units)
                    self.addUnitToTRF(unit)
                    unit.inputdata = copy.deepcopy(self.inputdata[id])
                    unit.inputdata.files = []
                    unit.inputdata.files += new_data.files

        elif self.mc_num_units > 0:

            if len(self.units) == 0:
                # check for appropriate splitter
                from GPI import GaussSplitter
                if not self.splitter or isType(self.splitter, GaussSplitter):
                    logger.warning(
                        "No GaussSplitter specified - first event info ignored"
                    )

                # create units for MC generation
                for i in range(0, self.mc_num_units):
                    unit = LHCbUnit()
                    unit.name = "Unit %d" % len(self.units)
                    self.addUnitToTRF(unit)
        else:
            import traceback
            traceback.print_stack()
            logger.error(
                "Please specify either inputdata or MC info for unit generation"
            )