def fetchDLSInfo(self): """ Contact DLS """ # Force show_prod=1 for everybody, grid jobs rely on central black list and t1access role to limit access to T1's self.cfg_params['CMSSW.show_prod'] = 1 # make assumption that same host won't be used for both # this check should catch most deployed servers (isDbs2, isDbs3, dbs2_url, dbs3_url) = verify_dbs_url(self) dbs_url=dbs3_url global_dbs3 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" # first try PhEDEx DLS_type="DLS_TYPE_PHEDEX" dls=DLSInfo(DLS_type,self.cfg_params) blockSites = dls.getReplicasBulk(self.Listfileblocks) if len(blockSites) == 0 : common.logger.info("No dataset location information found in PhEDEx") if dbs_url == global_dbs3: common.logger.info("Dataset in global DBS without location information") else: common.logger.info("Use origin site location recorded in local scope DBS") try: blockSites = self.getBlockSitesFromLocalDBS3(dbs_url) except: msg = "CAN'T GET LOCATION INFO FROM DBS END POINT: %s\n" % dbs_url raise CrabException(msg) self.SelectedSites = blockSites
def fetchDLSInfo(self): """ Contact DLS """ dlstype='' useDBS = self.cfg_params.get('CMSSW.dbs_url',None) scheduler = self.cfg_params.get('CRAB.scheduler',None).lower() global_url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" # Force show_prod=1 for everybody, grid jobs rely on central black list and t1access role to limit access to T1's # Don't switch to DBS just because user specified the global URL in config self.cfg_params['CMSSW.show_prod'] = 1 if useDBS == global_url: useDBS = None if useDBS: dlstype='dbs' DLS_type="DLS_TYPE_%s"%dlstype.upper() dls=DLSInfo(DLS_type,self.cfg_params) blockSites = self.PrepareDict(dls) else: dlstype='phedex' DLS_type="DLS_TYPE_%s"%dlstype.upper() dls=DLSInfo(DLS_type,self.cfg_params) blockSites = dls.getReplicasBulk(self.Listfileblocks) self.SelectedSites = blockSites
def fetchDLSInfo(self): """ Contact DLS """ dlstype='' useDBS = self.cfg_params.get('CMSSW.dbs_url',None) scheduler = self.cfg_params.get('CRAB.scheduler',None).lower() global_url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" # Condor must use show_prod = 1 to find data at local site which may be a Tier1 # Don't switch to DBS just because user specified the global URL in config if scheduler in ['condor']: self.cfg_params['CMSSW.show_prod'] = 1 if useDBS == global_url: useDBS = None if useDBS: dlstype='dbs' DLS_type="DLS_TYPE_%s"%dlstype.upper() dls=DLSInfo(DLS_type,self.cfg_params) blockSites = self.PrepareDict(dls) else: dlstype='phedex' DLS_type="DLS_TYPE_%s"%dlstype.upper() dls=DLSInfo(DLS_type,self.cfg_params) blockSites = dls.getReplicasBulk(self.Listfileblocks) self.SelectedSites = blockSites
def fetchDLSInfo(self): """ Contact DLS """ # make assumption that same host won't be used for both # this check should catch most deployed servers DBS2HOST = 'cmsdbsprod.cern.ch' DBS3HOST = 'cmsweb.cern.ch' useDBS2 = False useDBS3 = False useDAS = False global_dbs2 = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" global_dbs3 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" if self.cfg_params.get('CMSSW.use_dbs3'): useDBS3 = int(self.cfg_params.get('CMSSW.use_dbs3'))==1 if useDBS3: dbs_url= self.cfg_params.get('CMSSW.dbs_url', global_dbs3) else: dbs_url= self.cfg_params.get('CMSSW.dbs_url', global_dbs2) # Force show_prod=1 for everybody, grid jobs rely on central black list and t1access role to limit access to T1's self.cfg_params['CMSSW.show_prod'] = 1 if dbs_url==global_dbs2 or dbs_url==global_dbs3: # global DBS has no location info DLS_type="DLS_TYPE_PHEDEX" dls=DLSInfo(DLS_type,self.cfg_params) blockSites = dls.getReplicasBulk(self.Listfileblocks) else: # assume it is some local scope DBS dbs_endpoint = urlparse.urlsplit(dbs_url) if 'cmsdbsprod' in dbs_endpoint.hostname : DLS_type="DLS_TYPE_DBS" dls=DLSInfo(DLS_type,self.cfg_params) blockSites = self.PrepareDict(dls) elif 'cmsweb' in dbs_endpoint.hostname : blockSites = self.getBlockSitesFromLocalDBS3(dbs_url) else: # assume it is some test DBS3 end point try: blockSites = self.getBlockSitesFromLocalDBS3(dbs_url) except: msg = "CAN'T GET LOCATION INFO FROM DBS END POINT: %s\n" % dbs_url raise CrabException(msg) self.SelectedSites = blockSites