コード例 #1
0
ファイル: AnaTransform.py プロジェクト: slangrock/ganga
 def initialize(self):
     super(AnaTransform, self).initialize()
     self.application = AthenaTask()
     self.inputdata = DQ2Dataset()
コード例 #2
0
 def initialize(self):
    super(AnaTransform, self).initialize()
    self.application = AthenaTask()
    self.inputdata = DQ2Dataset()
コード例 #3
0
ファイル: AnaTransform.py プロジェクト: slangrock/ganga
class AnaTransform(Transform):
    """ Analyzes Events """
    _schema = Schema(
        Version(1, 0),
        dict(
            Transform._schema.datadict.items() + {
                'files_per_job':
                SimpleItem(defvalue=5, doc='files per job', modelist=["int"]),
                'partitions_data':
                ComponentItem('datasets',
                              defvalue=[],
                              optional=1,
                              sequence=1,
                              hidden=1,
                              doc='Input dataset for each partition'),
                'partitions_sites':
                SimpleItem(defvalue=[],
                           hidden=1,
                           modelist=["str", "list"],
                           doc='Input site for each partition'),
                'outputdata':
                ComponentItem('datasets',
                              defvalue=DQ2OutputDataset(),
                              doc='Output dataset'),
                'dataset_name':
                SimpleItem(defvalue="",
                           transient=1,
                           comparable=False,
                           getter="get_dataset_name",
                           doc='name of the output dataset'),
            }.items()))
    _category = 'transforms'
    _name = 'AnaTransform'
    _exportmethods = Transform._exportmethods

    def initialize(self):
        super(AnaTransform, self).initialize()
        self.application = AthenaTask()
        self.inputdata = DQ2Dataset()

    def get_dataset_name(self):
        task = self._getParent()
        name_base = [
            "user",
            getNickname(), task.creation_date,
            "task_%s" % task.id
        ]
        if self.inputdata.dataset:
            subtask_dsname = ".".join(name_base + [
                "subtask_%s" % task.transforms.index(self),
                str(self.inputdata.dataset[0].strip("/"))
            ])
        else:
            subtask_dsname = ".".join(
                name_base + ["subtask_%s" % task.transforms.index(self)])

        # make sure we keep the name size limit:
        dq2_config = getConfig("DQ2")
        max_length_site = len("ALBERTA-WESTGRID-T2_SCRATCHDISK      ")
        max_length = dq2_config['OUTPUTDATASET_NAMELENGTH'] - max_length_site
        if len(subtask_dsname) > max_length:
            logger.debug(
                "Proposed dataset name longer than limit (%d). Restricting dataset name..."
                % max_length)

            while len(subtask_dsname) > max_length:
                subtask_dsname_toks = subtask_dsname.split('.')
                subtask_dsname = '.'.join(
                    subtask_dsname_toks[:len(subtask_dsname_toks) - 1])
        return subtask_dsname


## Internal methods

    def checkCompletedApp(self, app):
        task = self._getParent()
        j = app._getParent()
        for odat in j.outputdata.outputdata:
            # Look out: if this is changed, there is anothher one like it below!
            if 0 == len([
                    f for f in j.outputdata.output
                    if ".".join(odat.split(".")[:-1]) in f
            ]):
                logger.error("Job %s has not produced %s file, only: %s" %
                             (j.id, odat, j.outputdata.output))
                return False
        # if this is the first app to complete the partition...
        if self.getPartitionStatus(self._app_partition[app.id]) != "completed":
            task_container, subtask_dsname = task.container_name, self.dataset_name

            infos = {}
            for oinfo in j.outputdata.output:
                try:
                    dq2_lock.acquire()
                    info = oinfo.split(",")
                    # get master replica from dataset - info not set to SE; but to ANALY_XYZ from panda
                    master_replica = dq2.getMasterReplicaLocation(info[0])
                    if master_replica:
                        info[5] = master_replica
                    else:
                        replicas = dq2.listDatasetReplicas(info[0]).values()
                        if len(replicas) == 0:
                            try:
                                info[5] = getPandaClient().PandaSites[
                                    info[5]]["ddm"]
                            except KeyError:
                                pass
                        else:
                            complete, incomplete = replicas[0].values()
                            info[5] = (complete + incomplete)[0]
                    if info[4][:3] == "ad:":
                        info[4] = info[4][3:]

                finally:
                    dq2_lock.release()

                datasetname = subtask_dsname + '.' + info[5]
                info[0] = datasetname
                infos.setdefault(datasetname, []).append(",".join(info))

            for ds in infos.keys():
                outputdata = DQ2OutputDataset()
                try:
                    outputdata.create_dataset(ds)
                except DQDatasetExistsException:
                    pass
                try:
                    outputdata.register_datasets_details(None, infos[ds])
                except DQFileExistsInDatasetException:
                    pass

            # Register Container
            try:
                containerinfo = {}
                dq2_lock.acquire()
                try:
                    containerinfo = dq2.listDatasets(task_container)
                except:
                    containerinfo = {}
                if containerinfo == {}:
                    try:
                        dq2.registerContainer(task_container)
                        logger.debug('Registered container for Task %i: %s' %
                                     (task.id, task_container))
                    except Exception as x:
                        logger.error(
                            'Problem registering container for Task %i, %s : %s %s'
                            % (task.id, task_container, x.__class__, x))
                for ds in infos.keys():
                    try:
                        dq2.registerDatasetsInContainer(task_container, [ds])
                    except DQContainerAlreadyHasDataset:
                        pass
                    except Exception as x:
                        logger.error(
                            'Problem registering dataset %s in container %s: %s %s'
                            % (subtask_dsname, task_container, x.__class__, x))
            finally:
                dq2_lock.release()
        return True

    def rebrokerPanda(self, cloud=None):
        sites = getPandaClient().PandaSites.keys()

        dataset_sites = [
            getPandaClient().convertDQ2toPandaID(site)
            for site in self.inputdata.get_locations(complete=1)
        ]
        sites = [site for site in sites if site in dataset_sites]
        sites = [
            s for s in sites
            if not s in self.backend.requirements.excluded_sites
        ]
        sites = [
            s for s in sites if not s.replace("ANALY_", "") in
            self.backend.requirements.excluded_sites
        ]
        if cloud:
            sites = [
                s for s in sites if s.replace("ANALY_", "") in
                getPandaClient().PandaClouds[cloud]["sites"]
            ]
        if len(sites) == 0:
            logger.error("No compatible sites for rebrokering found!")
            return

        from random import shuffle
        shuffle(sites)
        self.backend.site = sites[0]
        self.partitions_sites = [sites[0]] * len(self.partitions_sites)
        logger.warning(
            "Rebrokering transform - %i possible sites; chosing %s at random..."
            % (len(sites), sites[0]))

    def findCompleteCloudBackend(self, db_sites, allowed_sites, replicas):
        # Sort complete replicas into clouds
        # returns sorted list of tuples (cloud, backend)
        addclouds = TiersOfATLAS.ToACache.dbcloud.keys()
        random.shuffle(addclouds)
        clouds = config["cloudPreference"] + addclouds
        complete_sites = {}
        for c in clouds:
            complete_sites[c] = []
        for s in replicas[1]:
            try:
                cloud = whichCloudExt(s)
            except:
                logger.warning("Could not get cloud of site %s!", s)
                continue
            if db_sites and not stripSite(s) in db_sites:
                continue
            #print cloud, s
            complete_sites[cloud].append(s)

        result = []
        # Check if we find a cloud/backend combination for a complete replica
        backends = [
            be for be in config["backendPreference"] if be in GPI.__dict__
        ]
        for cloud in clouds:
            for backend in backends:
                if not backend in allowed_sites.keys():
                    continue
                if backend == "Panda":
                    sites = [
                        site for site in complete_sites[cloud]
                        if getPandaClient().convertDQ2toPandaID(site) in
                        allowed_sites["Panda"]
                    ]
                else:
                    sites = [
                        site for site in complete_sites[cloud]
                        if site in allowed_sites[backend]
                    ]
                if len(sites) > 0:
                    result.append((cloud, backend))
        return result

    def findIncompleteCloudBackend(self, db_sites, allowed_sites, replicas):
        # If no cloud/backend combination is found for complete replicas,
        # find cloud/backend with maximal number of replicas
        # returns list of tuples sorted by number of sites with replicas: (cloud, backend)
        addclouds = TiersOfATLAS.ToACache.dbcloud.keys()
        random.shuffle(addclouds)
        clouds = config["cloudPreference"] + addclouds
        incomplete_sites = {}
        for c in clouds:
            incomplete_sites[c] = []
        for s in replicas[0]:
            try:
                cloud = whichCloudExt(s)
            except:
                continue
            if db_sites and not stripSite(s) in db_sites:
                continue
            incomplete_sites[cloud].append(s)

        using_backend = None
        using_cloud = None
        max_sites = 0
        result = []
        backends = [
            be for be in config["backendPreference"] if be in GPI.__dict__
        ]
        for cloud in clouds:
            for backend in backends:
                if backend == "Panda":
                    from pandatools import Client
                    sites = [
                        site for site in incomplete_sites[cloud]
                        if getPandaClient().convertDQ2toPandaID(site) in
                        allowed_sites["Panda"]
                    ]
                else:
                    sites = [
                        site for site in incomplete_sites[cloud]
                        if site in allowed_sites[backend]
                    ]
                if len(sites) > 0:
                    result.append((len(sites), cloud, backend))
        result.sort()
        return [(r[1], r[2]) for r in result]

    def check(self):
        super(AnaTransform, self).check()
        if not self.inputdata.dataset:
            return
        if not self.backend:
            logger.warning("Determining backend and cloud...")

            # Get ddm sites of atlas_dbrelease, if present
            db_sites = None
            if self.application.atlas_dbrelease == "LATEST":
                from pandatools import Client
                self.application.atlas_dbrelease = getPandaClient(
                ).getLatestDBRelease(False)
            if self.application.atlas_dbrelease:
                try:
                    db_dataset = self.application.atlas_dbrelease.split(':')[0]
                    try:
                        dq2_lock.acquire()
                        db_locations = dq2.listDatasetReplicas(
                            db_dataset).values()[0][1]
                    finally:
                        dq2_lock.release()

                except Exception as x:
                    raise ApplicationConfigurationError(
                        x,
                        'Problem in AnaTask - j.application.atlas_dbrelease is wrongly configured ! '
                    )
                db_sites = stripSites(db_locations)

            # Get complete/incomplete ddm sites for input dataset
            ds = self.inputdata.dataset[0]
            try:
                dq2_lock.acquire()
                if ds[-1] != "/":
                    try:
                        replicas = {ds: dq2.listDatasetReplicas(ds)}
                    except DQUnknownDatasetException:
                        ds += "/"
                if ds[-1] == "/":
                    replicas = dq2.listDatasetReplicasInContainer(ds)
            finally:
                dq2_lock.release()

            # check if replicas are non-empty
            somefound = False
            for tid in replicas:
                if len(replicas[tid]) == 0:
                    raise ApplicationConfigurationError(
                        None, "No replicas for dataset %s found!" % tid)
            replicas = [r.values()[0] for r in replicas.values()
                        ]  # (dict with only one entry)

            # Get allowed sites for each backend:
            backends = [
                be for be in config["backendPreference"] if be in GPI.__dict__
            ]
            allowed_sites = {}
            if "LCG" in backends:
                allowed_sites["LCG"] = GPI.LCG().requirements.list_sites(
                    True, True)
            if "Panda" in backends:
                from pandatools import Client
                allowed_sites["Panda"] = getPandaClient().PandaSites.keys()
                #allowed_sites["Panda"] = [site["ddm"] for site in Client.getSiteSpecs()[1].values()]
            if "NG" in backends:
                allowed_sites["NG"] = getConfig(
                    "Athena")["AllowedSitesNGDQ2JobSplitter"]
            #if "PBS" in backends:
            #   sites["PBS"] = [] # should be local DQ2 storage element!

            # Get list of cloud-backend pairs (cbl) for complete replicas
            common_cbl = None
            for r in replicas:
                cbl = self.findCompleteCloudBackend(db_sites, allowed_sites, r)
                if common_cbl is None:
                    common_cbl = cbl
                else:
                    common_cbl = [cb for cb in cbl if cb in common_cbl]

            #print "CLOUD/BACKEND list for COMPLETE replicas: ", common_cbl

            # ..and for incomplete replicas
            if common_cbl is None or len(common_cbl) == 0:
                if len(replicas) > 1:
                    raise ApplicationConfigurationError(
                        None,
                        'Container dataset %s has no complete replica on one site and backend. Please specify individual tid datasets or use t.initializeFromDataset("%s") '
                        % (ds, ds))
                common_cbl = self.findIncompleteCloudBackend(
                    db_sites, allowed_sites, replicas[0])
                #print "CLOUD/BACKEND list for INCOMPLETE replicas: ", common_cbl
            if common_cbl is None or len(common_cbl) == 0:
                raise ApplicationConfigurationError(
                    None,
                    'Container dataset %s has no replica on one site and backend. Please specify individual tid datasets!'
                    % (ds))

            cb = common_cbl[0]
            using_cloud = cb[0]
            using_backend = cb[1]

            assert using_cloud, using_backend

            if using_backend == "Panda":
                self.backend = stripProxy(GPI.Panda())
                self.backend.requirements.cloud = using_cloud

            elif using_backend == "NG":
                self.backend = stripProxy(GPI.NG())
            elif using_backend == "LCG":
                self.backend = stripProxy(GPI.LCG())
                self.backend.requirements = stripProxy(
                    GPI.AtlasLCGRequirements())
                self.backend.requirements.cloud = using_cloud
            assert self.backend
            logger.warning("Running on cloud %s using backend %s", using_cloud,
                           using_backend)

        logger.warning("Determining partition splitting...")
        try:
            if not self.backend.requirements.cloud:
                self.backend.requirements.cloud = "DE"
        except:
            pass
        if not self.inputdata.dataset:
            return
        splitter = DQ2JobSplitter()
        splitter.numfiles = self.files_per_job
        #splitter.update_siteindex = False # commented to use default value
        #splitter.use_lfc = True
        sjl = splitter.split(
            self
        )  # This works even for Panda, no special "Job" properties are used anywhere.
        self.partitions_data = [sj.inputdata for sj in sjl]
        try:
            self.partitions_sites = [
                sj.backend.requirements.sites for sj in sjl
            ]
        except AttributeError:
            self.partitions_sites = [sj.backend.site for sj in sjl]
            pass
        self.setPartitionsLimit(len(self.partitions_data) + 1)
        self.setPartitionsStatus([
            c for c in range(1,
                             len(self.partitions_data) + 1)
            if self.getPartitionStatus(c) != "completed"
        ], "ready")

    def getJobsForPartitions(self, partitions):
        j = self.createNewJob(partitions[0])
        if len(partitions) >= 1:
            j.splitter = AnaTaskSplitterJob()
            j.splitter.subjobs = partitions
        j.inputdata = self.partitions_data[partitions[0] - 1]
        if self.partitions_sites:
            if stripProxy(j.backend)._name == "Panda":
                if j.backend.site == "AUTO":
                    j.backend.site = self.partitions_sites[partitions[0] - 1]
            else:
                j.backend.requirements.sites = self.partitions_sites[
                    partitions[0] - 1]
        j.outputdata = self.outputdata
        if stripProxy(
                j.backend
        )._name == "Panda" and j.application.atlas_exetype == "ATHENA":
            j.outputdata.outputdata = []
        #j.outputdata.datasetname = ""
        task = self._getParent()
        dsn = [
            "user",
            getNickname(), task.creation_date,
            "%i.t_%s_%s" % (j.id, task.id, task.transforms.index(self))
        ]
        j.outputdata.datasetname = ".".join(dsn)

        #if j.outputdata.datasetname:
        #today = time.strftime("%Y%m%d",time.localtime())
        #j.outputdata.datasetname = "%s.%i.%s" % (j.outputdata.datasetname, j.id, today)
        return [j]

    def info(self):
        print markup("%s '%s'" % (self.__class__.__name__, self.name),
                     status_colours[self.status])
        print "* dataset: %s " % self.inputdata.dataset
        print "* processing %s per job" % say(self.files_per_job, "file")
        print "* backend: %s" % self.backend.__class__.__name__
        print "* application:"
        self.application.printTree()
コード例 #4
0
class AnaTransform(Transform):
   """ Analyzes Events """
   _schema = Schema(Version(1,0), dict(Transform._schema.datadict.items() + {
       'files_per_job'     : SimpleItem(defvalue=5, doc='files per job', modelist=["int"]),
       'partitions_data'   : ComponentItem('datasets', defvalue=[], optional=1, sequence=1, hidden=1, doc='Input dataset for each partition'),
       'partitions_sites'  : SimpleItem(defvalue=[], hidden=1, modelist=["str","list"],doc='Input site for each partition'),
       'outputdata'        : ComponentItem('datasets', defvalue=DQ2OutputDataset(), doc='Output dataset'),
       'dataset_name'      : SimpleItem(defvalue="", transient=1, comparable=False, getter="get_dataset_name", doc='name of the output dataset'),
       }.items()))
   _category = 'transforms'
   _name = 'AnaTransform'
   _exportmethods = Transform._exportmethods

   def initialize(self):
      super(AnaTransform, self).initialize()
      self.application = AthenaTask()
      self.inputdata = DQ2Dataset()

   def get_dataset_name(self):
      task = self._getParent()
      name_base = ["user",getNickname(),task.creation_date,"task_%s" % task.id]
      if self.inputdata.dataset:
          subtask_dsname = ".".join(name_base +["subtask_%s" % task.transforms.index(self), str(self.inputdata.dataset[0].strip("/"))])
      else:
          subtask_dsname = ".".join(name_base +["subtask_%s" % task.transforms.index(self)])
            
      # make sure we keep the name size limit:
      dq2_config = getConfig("DQ2")
      max_length_site = len("ALBERTA-WESTGRID-T2_SCRATCHDISK      ")
      max_length = dq2_config['OUTPUTDATASET_NAMELENGTH'] - max_length_site
      if len(subtask_dsname) > max_length:
          logger.debug("Proposed dataset name longer than limit (%d). Restricting dataset name..." % max_length)

          while len(subtask_dsname) > max_length:
              subtask_dsname_toks = subtask_dsname.split('.')
              subtask_dsname = '.'.join(subtask_dsname_toks[:len(subtask_dsname_toks)-1])
      return subtask_dsname

## Internal methods
   def checkCompletedApp(self, app):
      task = self._getParent()
      j = app._getParent()
      for odat in j.outputdata.outputdata:
          # Look out: if this is changed, there is anothher one like it below!
          if 0==len([f for f in j.outputdata.output if ".".join(odat.split(".")[:-1]) in f]):
              logger.error("Job %s has not produced %s file, only: %s" % (j.id, odat, j.outputdata.output))
              return False
      # if this is the first app to complete the partition...
      if self.getPartitionStatus(self._app_partition[app.id]) != "completed":
          task_container, subtask_dsname = task.container_name, self.dataset_name

          infos = {}
          for oinfo in j.outputdata.output:
              try:
                  dq2_lock.acquire()
                  info = oinfo.split(",")
                  # get master replica from dataset - info not set to SE; but to ANALY_XYZ from panda
                  master_replica = dq2.getMasterReplicaLocation(info[0])
                  if master_replica:
                      info[5] = master_replica
                  else:
                      replicas = dq2.listDatasetReplicas(info[0]).values()
                      if len(replicas) == 0:
                          try:
                              info[5] = getPandaClient().PandaSites[info[5]]["ddm"]
                          except KeyError:
                              pass
                      else:
                          complete, incomplete = replicas[0].values()
                          info[5] = (complete + incomplete)[0]
                  if info[4][:3] == "ad:":
                      info[4] = info[4][3:]

              finally:
                  dq2_lock.release()
                
              datasetname = subtask_dsname + '.' + info[5]
              info[0] = datasetname
              infos.setdefault(datasetname, []).append(",".join(info))

          for ds in infos.keys():
              outputdata = DQ2OutputDataset()
              try:
                  outputdata.create_dataset(ds)
              except DQDatasetExistsException:
                  pass
              try:
                  outputdata.register_datasets_details(None, infos[ds])
              except DQFileExistsInDatasetException:
                  pass

          # Register Container
          try:
              containerinfo = {}
              dq2_lock.acquire()
              try:
                  containerinfo = dq2.listDatasets(task_container)
              except:
                  containerinfo = {}
              if containerinfo == {}:
                  try:
                      dq2.registerContainer(task_container)
                      logger.debug('Registered container for Task %i: %s' % (task.id, task_container))
                  except Exception as x:
                      logger.error('Problem registering container for Task %i, %s : %s %s' % (task.id, task_container,x.__class__, x))
              for ds in infos.keys():
                  try:
                      dq2.registerDatasetsInContainer(task_container, [ ds ] )
                  except DQContainerAlreadyHasDataset:
                      pass
                  except Exception as x:
                      logger.error('Problem registering dataset %s in container %s: %s %s' %( subtask_dsname, task_container, x.__class__, x))
          finally:
              dq2_lock.release()
      return True

   def rebrokerPanda(self, cloud = None):
      sites = getPandaClient().PandaSites.keys()

      dataset_sites = [getPandaClient().convertDQ2toPandaID(site) for site in self.inputdata.get_locations(complete=1)]
      sites = [site for site in sites if site in dataset_sites]
      sites = [s for s in sites if not s in self.backend.requirements.excluded_sites] 
      sites = [s for s in sites if not s.replace("ANALY_","") in self.backend.requirements.excluded_sites]
      if cloud:
         sites = [s for s in sites if s.replace("ANALY_","") in getPandaClient().PandaClouds[cloud]["sites"]] 
      if len(sites) == 0:
         logger.error("No compatible sites for rebrokering found!")
         return

      from random import shuffle
      shuffle(sites)
      self.backend.site = sites[0]
      self.partitions_sites = [sites[0]]*len(self.partitions_sites)
      logger.warning("Rebrokering transform - %i possible sites; chosing %s at random..." % (len(sites),sites[0]))


   def findCompleteCloudBackend(self,db_sites,allowed_sites,replicas):
      # Sort complete replicas into clouds
      # returns sorted list of tuples (cloud, backend)
      addclouds = TiersOfATLAS.ToACache.dbcloud.keys()
      random.shuffle(addclouds)
      clouds = config["cloudPreference"] + addclouds
      complete_sites = {}
      for c in clouds:
         complete_sites[c] = []
      for s in replicas[1]:
         try:
            cloud = whichCloudExt(s)
         except:
            logger.warning("Could not get cloud of site %s!", s)
            continue
         if db_sites and not stripSite(s) in db_sites:
            continue
         #print cloud, s
         complete_sites[cloud].append(s)

      result = []
      # Check if we find a cloud/backend combination for a complete replica
      backends = [be for be in config["backendPreference"] if be in GPI.__dict__]
      for cloud in clouds:
         for backend in backends:
            if not backend in allowed_sites.keys(): 
               continue
            if backend == "Panda":
                sites = [site for site in complete_sites[cloud] if getPandaClient().convertDQ2toPandaID(site) in allowed_sites["Panda"]]
            else:
                sites = [site for site in complete_sites[cloud] if site in allowed_sites[backend]]
            if len(sites) > 0:
               result.append((cloud,backend))
      return result
 
   def findIncompleteCloudBackend(self,db_sites,allowed_sites,replicas):
      # If no cloud/backend combination is found for complete replicas, 
      # find cloud/backend with maximal number of replicas
      # returns list of tuples sorted by number of sites with replicas: (cloud, backend)
      addclouds = TiersOfATLAS.ToACache.dbcloud.keys()
      random.shuffle(addclouds)
      clouds = config["cloudPreference"] + addclouds
      incomplete_sites = {}
      for c in clouds:
         incomplete_sites[c] = []
      for s in replicas[0]:
         try:
            cloud = whichCloudExt(s)
         except:
            continue
         if db_sites and not stripSite(s) in db_sites:
            continue
         incomplete_sites[cloud].append(s)
      
      using_backend = None
      using_cloud = None
      max_sites = 0
      result = []
      backends = [be for be in config["backendPreference"] if be in GPI.__dict__]
      for cloud in clouds:
         for backend in backends:
            if backend == "Panda":
                from pandatools import Client
                sites = [site for site in incomplete_sites[cloud] if getPandaClient().convertDQ2toPandaID(site) in allowed_sites["Panda"]]
            else:
                sites = [site for site in incomplete_sites[cloud] if site in allowed_sites[backend]]
            if len(sites) > 0:
               result.append((len(sites),cloud,backend))
      result.sort()
      return [(r[1],r[2]) for r in result]

   def check(self):
      super(AnaTransform,self).check()
      if not self.inputdata.dataset:
         return
      if not self.backend:
         logger.warning("Determining backend and cloud...")

         # Get ddm sites of atlas_dbrelease, if present
         db_sites = None
         if self.application.atlas_dbrelease == "LATEST":
            from pandatools import Client
            self.application.atlas_dbrelease = getPandaClient().getLatestDBRelease(False)
         if self.application.atlas_dbrelease:
            try:
               db_dataset = self.application.atlas_dbrelease.split(':')[0] 
               try:
                  dq2_lock.acquire()
                  db_locations = dq2.listDatasetReplicas(db_dataset).values()[0][1] 
               finally:
                  dq2_lock.release()

            except Exception as x:
               raise ApplicationConfigurationError(x, 'Problem in AnaTask - j.application.atlas_dbrelease is wrongly configured ! ')
            db_sites = stripSites(db_locations)
         
         # Get complete/incomplete ddm sites for input dataset
         ds = self.inputdata.dataset[0]
         try:
            dq2_lock.acquire()
            if ds[-1] != "/":
               try:
                  replicas = {ds : dq2.listDatasetReplicas(ds)}
               except DQUnknownDatasetException:
                  ds += "/"
            if ds[-1] == "/":
               replicas = dq2.listDatasetReplicasInContainer(ds)
         finally:
            dq2_lock.release()
        
         # check if replicas are non-empty
         somefound = False
         for tid in replicas: 
            if len(replicas[tid]) == 0:
               raise ApplicationConfigurationError(None, "No replicas for dataset %s found!" % tid)
         replicas = [r.values()[0] for r in replicas.values()] # (dict with only one entry)

         # Get allowed sites for each backend:
         backends = [be for be in config["backendPreference"] if be in GPI.__dict__]
         allowed_sites = {}
         if "LCG" in backends:
            allowed_sites["LCG"] = GPI.LCG().requirements.list_sites(True,True)
         if "Panda" in backends:
            from pandatools import Client
            allowed_sites["Panda"] = getPandaClient().PandaSites.keys()
            #allowed_sites["Panda"] = [site["ddm"] for site in Client.getSiteSpecs()[1].values()]
         if "NG" in backends:
            allowed_sites["NG"] = getConfig("Athena")["AllowedSitesNGDQ2JobSplitter"]
         #if "PBS" in backends:
         #   sites["PBS"] = [] # should be local DQ2 storage element!

         # Get list of cloud-backend pairs (cbl) for complete replicas
         common_cbl = None
         for r in replicas:
            cbl = self.findCompleteCloudBackend(db_sites, allowed_sites, r)
            if common_cbl is None:
               common_cbl = cbl
            else:
               common_cbl = [cb for cb in cbl if cb in common_cbl]

         #print "CLOUD/BACKEND list for COMPLETE replicas: ", common_cbl

         # ..and for incomplete replicas
         if common_cbl is None or len(common_cbl) == 0:
            if len(replicas) > 1:
               raise ApplicationConfigurationError(None, 'Container dataset %s has no complete replica on one site and backend. Please specify individual tid datasets or use t.initializeFromDataset("%s") ' % (ds, ds))
            common_cbl = self.findIncompleteCloudBackend(db_sites, allowed_sites, replicas[0])
            #print "CLOUD/BACKEND list for INCOMPLETE replicas: ", common_cbl
         if common_cbl is None or len(common_cbl) == 0:
            raise ApplicationConfigurationError(None, 'Container dataset %s has no replica on one site and backend. Please specify individual tid datasets!' % (ds))

         cb = common_cbl[0]
         using_cloud = cb[0]
         using_backend = cb[1]

         assert using_cloud, using_backend

         if using_backend == "Panda":
            self.backend = stripProxy(GPI.Panda())
            self.backend.requirements.cloud = using_cloud

         elif using_backend == "NG":
            self.backend = stripProxy(GPI.NG())
         elif using_backend == "LCG":
            self.backend = stripProxy(GPI.LCG())
            self.backend.requirements = stripProxy(GPI.AtlasLCGRequirements())
            self.backend.requirements.cloud = using_cloud
         assert self.backend
         logger.warning("Running on cloud %s using backend %s", using_cloud, using_backend)

      logger.warning("Determining partition splitting...")
      try:
         if not self.backend.requirements.cloud:
            self.backend.requirements.cloud = "DE"
      except:
         pass
      if not self.inputdata.dataset:
         return
      splitter = DQ2JobSplitter()
      splitter.numfiles = self.files_per_job
      #splitter.update_siteindex = False # commented to use default value
      #splitter.use_lfc = True
      sjl = splitter.split(self) # This works even for Panda, no special "Job" properties are used anywhere.
      self.partitions_data = [sj.inputdata for sj in sjl]
      try:
         self.partitions_sites = [sj.backend.requirements.sites for sj in sjl]
      except AttributeError:
         self.partitions_sites = [sj.backend.site for sj in sjl]
         pass
      self.setPartitionsLimit(len(self.partitions_data)+1)
      self.setPartitionsStatus([c for c in range(1,len(self.partitions_data)+1) if self.getPartitionStatus(c) != "completed"], "ready")
   
   def getJobsForPartitions(self, partitions):
      j = self.createNewJob(partitions[0])
      if len(partitions) >= 1:
          j.splitter = AnaTaskSplitterJob()
          j.splitter.subjobs = partitions
      j.inputdata = self.partitions_data[partitions[0]-1]
      if self.partitions_sites:
         if stripProxy(j.backend)._name == "Panda":
            if j.backend.site == "AUTO":
               j.backend.site = self.partitions_sites[partitions[0]-1]
         else:
            j.backend.requirements.sites = self.partitions_sites[partitions[0]-1]
      j.outputdata = self.outputdata
      if stripProxy(j.backend)._name == "Panda" and j.application.atlas_exetype == "ATHENA":
          j.outputdata.outputdata=[]
      #j.outputdata.datasetname = ""
      task = self._getParent()
      dsn = ["user",getNickname(),task.creation_date,"%i.t_%s_%s" % (j.id, task.id, task.transforms.index(self))]
      j.outputdata.datasetname = ".".join(dsn)

      #if j.outputdata.datasetname:
         #today = time.strftime("%Y%m%d",time.localtime())
         #j.outputdata.datasetname = "%s.%i.%s" % (j.outputdata.datasetname, j.id, today)
      return [j]

   def info(self):
      print markup("%s '%s'" % (self.__class__.__name__, self.name), status_colours[self.status])
      print "* dataset: %s " % self.inputdata.dataset
      print "* processing %s per job" % say(self.files_per_job,"file")
      print "* backend: %s" % self.backend.__class__.__name__
      print "* application:"
      self.application.printTree()