def create_iface_binding(iface_type, endpoint, dbsConf, verbose): if iface_type == dlsClient.DLS_TYPE_PHEDEX: iface = dlsClient.getDlsApi(iface_type, endpoint, dbs_client_config=dbsConf, uaFlexString = 'DLS-CLI') else: iface = dlsClient.getDlsApi(iface_type, endpoint, dbs_client_config=dbsConf) if(verbose == 2): iface.setVerbosity(DLS_VERB_HIGH) else: if(verbose == 0): iface.setVerbosity(DLS_VERB_NONE) else: if(verbose == 1): iface.setVerbosity(DLS_VERB_WARN) return iface
def __init__(self, type, cfg_params): self.cfg_params = cfg_params self.showCAF = False self.showProd = False self.subscribed = False subscribed = int(self.cfg_params.get('CMSSW.subscribed', 0)) if subscribed == 1: self.subscribed = True showProd = int(self.cfg_params.get('CMSSW.show_prod', 0)) if showProd == 1: self.showProd = True phedexURL = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/' global_url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" if type == "DLS_TYPE_DBS": # use dbs_url as dls_endpoint if dls_type is dbs endpoint = self.cfg_params.get('CMSSW.dbs_url', global_url) elif type == "DLS_TYPE_PHEDEX": endpoint = self.cfg_params.get('CMSSW.dls_phedex_url', phedexURL) if self.cfg_params['CRAB.scheduler'].upper() == 'CAF': self.showCAF = True else: msg = "DLS type %s not among the supported DLS ( DLS_TYPE_DLI and DLS_TYPE_MYSQL ) " % type raise CrabException(msg) common.logger.debug("DLS interface: %s Server %s" % (type, endpoint)) try: self.api = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint, verbosity=dlsApi.DLS_VERB_INFO) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: %s Server %s" % ( str(inst), endpoint) #print msg raise CrabException(msg)
class DBS3Reader: """ _DBSReader_ General API for reading data from DBS """ def __init__(self, url, **contact): # instantiate dbs api object try: self.dbs = DbsApi(url, **contact) except DbsException, ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg) # setup DLS api dlsType = 'DLS_TYPE_PHEDEX' dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod' try: self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version='') except DlsApiError, ex: msg = "Error in DBSReader with DlsApi\n" msg += "%s\n" % str(ex) raise DBSReaderError(msg)
def __init__(self, url, **contact): args = {"url": url, "level": 'ERROR', "version": ''} args.update(contact) #try: self.dbs = DbsApi(args) #except DbsException, ex: # msg = "Error in DBSReader with DbsApi\n" # msg += "%s\n" % formatEx(ex) # raise DBSReaderError(msg) # setup DLS api - with either dbs or phedex depending on dbs instance if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \ self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL': dlsType = 'DLS_TYPE_PHEDEX' dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod' else: dlsType = 'DLS_TYPE_DBS' dlsUrl = url try: self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version=args['version']) except DlsApiError, ex: msg = "Error in DBSReader with DlsApi\n" msg += "%s\n" % str(ex) raise DBSReaderError(msg)
def __init__(self, type, cfg_params): self.cfg_params = cfg_params self.showCAF = False self.showProd = False self.subscribed = False subscribed =int(self.cfg_params.get('CMSSW.subscribed', 0)) if subscribed == 1: self.subscribed = True showProd = int(self.cfg_params.get('CMSSW.show_prod', 0)) if showProd == 1: self.showProd = True phedexURL='https://cmsweb.cern.ch/phedex/datasvc/xml/prod/' global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" if type=="DLS_TYPE_DBS": # use dbs_url as dls_endpoint if dls_type is dbs endpoint=self.cfg_params.get('CMSSW.dbs_url', global_url) elif type=="DLS_TYPE_PHEDEX": endpoint=self.cfg_params.get('CMSSW.dls_phedex_url',phedexURL) if self.cfg_params['CRAB.scheduler'].upper() == 'CAF': self.showCAF = True else: msg = "DLS type %s not among the supported DLS ( DLS_TYPE_DLI and DLS_TYPE_MYSQL ) "%type raise CrabException(msg) common.logger.debug("DLS interface: %s Server %s"%(type,endpoint)) try: self.api = dlsClient.getDlsApi(dls_type=type,dls_endpoint=endpoint,verbosity=dlsApi.DLS_VERB_INFO) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: %s Server %s"%(str(inst),endpoint) #print msg raise CrabException(msg)
def testF_GetBlocksOfLocationList(self): print "\n\n########## Testing the retrieval of fileblocks for several locations" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Get FileBlocks given the locations: %s, %s" % (seA, seB) entryList = api.getFileBlocks([self.C.locA, self.C.locB], errorTolerant=False) fList = [x.fileBlock.name for x in entryList] print "\nlen(fList): %s" % len(fList) self.assert_(len(fList) > self.C.lenListSeA)
def testB_ListOneFileblock(self): print "\n\n########## Testing the listing of one fileblock" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "List a DLS entry with fileblock=%s" % fbA fList = api.listFileBlocks(self.C.fblockA) print "\nfList: %s" % ([x.name for x in fList],) self.assert_(len(fList) == 1) self.assertEqual(fList[0].name, fbA)
def __init__(self, dls_type, dls_address): self.conf = {} self.conf['type'] = dls_type self.conf['address'] = dls_address try: logging.debug('Instantiating DLS client interface') self.api= dlsClient.getDlsApi(dls_type = dls_type,\ dls_endpoint = dls_address) logging.debug('DLS client interface instantiated') except dlsApi.DlsApiError, inst: raise ProdException(exceptions[4010] + str(inst), 4010)
def __init__(self,dls_type,dls_address): self.conf={} self.conf['type']=dls_type self.conf['address']=dls_address try: logging.debug('Instantiating DLS client interface') self.api= dlsClient.getDlsApi(dls_type = dls_type,\ dls_endpoint = dls_address) logging.debug('DLS client interface instantiated') except dlsApi.DlsApiError, inst: raise ProdException(exceptions[4010]+str(inst),4010)
def testC_ListFileblockPattern(self): print "\n\n########## Testing the listing of one fileblock and a pattern" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "List several DLS entries for pattern=%s and block=%s"%(fbPattern, fbB) fList = api.listFileBlocks([self.C.fblockPat, self.C.fblockB]) print "\nfList: %s" % ([x.name for x in fList],) self.assert_(len(fList) >= 2) self.assert_(fbA in [x.name for x in fList]) self.assert_(fbB in [x.name for x in fList]) # Store fileblock list for use at test H self.C.fListFbPat = fList
def testJ_GetAllLocations(self): print "\n\n########## Getting all existing locations" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) locList = api.getAllLocations() self.assert_(len(locList) > 20) locs = [x.host for x in locList] print "All locations: %s" % locs self.assert_(seA in locs) self.assert_(seB in locs) for loc in self.C.locsFblockSeA: self.assert_(loc in locs)
def testG_GetLocationOfOneBlockWithT01(self): print "\n\n########## Getting location of one fileblock including T0 and T1s" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Get location of fileblock=%s" % self.C.fblockSeA entryList = api.getLocations(self.C.fblockSeA, errorTolerant=False, showProd=True) self.assert_(len(entryList) == 1) locs = [x.host for x in entryList[0].locations] print "\nlocations: %s" % locs self.assert_(seA in locs) self.assert_(len(locs) > len(self.C.locsFblockSeA)) for loc in self.C.locsFblockSeA: self.assert_(loc in locs)
def testE_GetLocationOfOneBlock(self): print "\n\n########## Getting location of one fileblock" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Get location of fileblock=%s" % self.C.fblockSeA entryList = api.getLocations(self.C.fblockSeA, errorTolerant = False) self.assert_(len(entryList) == 1) locs = [x.host for x in entryList[0].locations] print "\nlocations: %s" % locs self.assert_(seA in locs) # Store the locations list to compare with those of tests G and I self.C.locsFblockSeA = locs
def testD_GetBlocksOfLocation(self): print "\n\n########## Testing the retrieval of fileblocks for one location" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Get FileBlocks given the location: %s" % (seA) entryList = api.getFileBlocks(self.C.locA, errorTolerant=False) fList = [x.fileBlock.name for x in entryList] print "\nlen(fList): %s" % len(fList) self.assert_(len(fList) > 1) # Store one fileblock at the given location for next test self.C.fblockSeA = fList[0] # Store the block list length to compare with test F self.C.lenListSeA = len(fList)
def testL_QueryWithUserAgent(self): print "\n\n########## Querying with user agent set" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT, uaFlexString = 'This is a test', uaClientsList = [['Tester', '1.1'], ['Sth', '2_4']]) print "Get location of fileblock=%s" % self.C.fblockSeA entryList = api.getLocations(self.C.fblockSeA, errorTolerant = False) self.assert_(len(entryList) == 1) locs = [x.host for x in entryList[0].locations] print "\nlocations: %s" % locs self.assert_(seA in locs) # Store the locations list to compare with those of tests G and I self.C.locsFblockSeA = locs
def testK_GetFileLocsForOneBlock(self): print "\n\n########## Getting file locs for one fileblock" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Get file locs for: %s" % self.C.fblockSeA fileList = api.getFileLocs(self.C.fblockSeA) self.assert_(len(fileList) == 1) block = fileList[0][0] self.assertEqual(block.name, self.C.fblockSeA) fileLocs = fileList[0][1] self.assert_(len(fileLocs) > 1) for f in fileLocs: for loc in fileLocs[f]: # print 'f, locs: %s, %s' % (f, fileLocs[f]) self.assert_(loc.host in self.C.locsFblockSeA)
def testI_GetLocationOfListOfBlocks(self): print "\n\n########## Getting locations of a list of fileblocks" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Get location of fileblocks: %s, %s" % (self.C.fblockSeA, fbA) entryList = api.getLocations([self.C.fblockSeA, self.C.fblockA], errorTolerant=False) for entry in entryList: block = entry.fileBlock.name locs = [x.host for x in entry.locations] print "\nlocations for block %s: %s" % (block, locs) if block == fbA: self.assert_(len(entry.locations) > 0) else: self.assertEqual(len(entry.locations), len(self.C.locsFblockSeA)) for loc in self.C.locsFblockSeA: self.assert_(loc in locs)
def testH_DumpEntriesWithPattern(self): print "\n\n########## Dump several entries for pattern" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT) print "Dumping entries for pattern: %s" % fbPattern entryList = api.dumpEntries(fbPattern, errorTolerant = False) self.assert_(len(entryList) > 1) fList = [x.fileBlock.name for x in entryList] print "\nfList: %s" % fList print "\nself.C.fListFbPat: %s" % [x.name for x in self.C.fListFbPat] # Check the fileblocks are the same as those listed before self.assertEqual(len(fList), len(self.C.fListFbPat)) for fb in fList: self.assert_(fb in [x.name for x in self.C.fListFbPat]) # Check each fileblock has a non-emtpy list of locations for entry in entryList: self.assert_(len(entry.locations) > 0)
def __init__(self, url, **contact): args = {"url": url, "level": "ERROR", "version": ""} args.update(contact) # try: self.dbs = DbsApi(args) # except DbsException, ex: # msg = "Error in DBSReader with DbsApi\n" # msg += "%s\n" % formatEx(ex) # raise DBSReaderError(msg) # setup DLS api - with either dbs or phedex depending on dbs instance if url.count("cmsdbsprod.cern.ch/cms_dbs_prod_global") or self.dbs.getServerInfo()["InstanceName"] == "GLOBAL": dlsType = "DLS_TYPE_PHEDEX" dlsUrl = "https://cmsweb.cern.ch/phedex/datasvc/xml/prod" else: dlsType = "DLS_TYPE_DBS" dlsUrl = url try: self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version=args["version"]) except DlsApiError, ex: msg = "Error in DBSReader with DlsApi\n" msg += "%s\n" % str(ex) raise DBSReaderError(msg)
hostname = 'localhost' pass msg ="At %s found %s %s processes on %s.\n"%(time.asctime(),nProc,procName,hostname) msg+="Please take some action, since it indicates that DLS is not responding.\n" msg+="The %s is a part of DBS data discovery and runs as cronjob on %s."%(procName,hostname) DBSUtil.sendEmail(msg) sys.exit(1) # get grid proxy # cmd="cat $HOME/.globus/pp.txt | grid-proxy-init -pwstdin -q" cmd="cat /data/DBSDataDiscovery/COMP/DBS/Web/DataDiscovery/pp.txt | grid-proxy-init -cert /data/vk/cert/usercert.pem -key /data/vk/cert/userkey.pem -pwstdin -q" os.system(cmd) file = open('dls.all.tmp','w') for dbsInst in DBSInst.DBS_DLS_INST.keys(): iface="" try: url,dlsType,endpoint = DBSInst.DBS_DLS_INST[dbsInst] if dlsType=='DLS_TYPE_DLI': dlsType='DLS_TYPE_LFC' iface = dlsClient.getDlsApi(dlsType,endpoint) #print dbsInst,dlsType,endpoint,iface locList = iface.getAllLocations(session = True) for loc in locList: file.write('%s %s\n'%(dbsInst,loc.host)) #DBSUtil.printListElements(lList,dbsInst) except: printExcept() raise "Fail to generate location list for dbs instance",dbsInst file.close() os.rename('dls.all.tmp','dls.all')
# // # // Local DLS instance to start from #// localDLSendpoint=MapDBSDLSinstance[localDBS] localDLStype='DLS_TYPE_LFC' if localDBS == 'MCLocal_1': localDLStype='DLS_TYPE_MYSQL' # // # // Local and Global DLS API #// print "" print " From Local DLS Server endpoint: %s (type: %s) "%(localDLSendpoint,localDLStype) try: localDLSapi = dlsClient.getDlsApi(dls_type=localDLStype,dls_endpoint=localDLSendpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit() print " to Global DLS Server endpoint: %s (type: %s)"%(endpoint,type) print "" try: globalDLSapi = dlsClient.getDlsApi(dls_type=type,dls_endpoint=endpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit()
return msg ##################################################333 # Unit testing if __name__ == "__main__": import dlsClient from dlsDataObjects import * ## use DLS server type="DLS_TYPE_MYSQL" server ="lxgate10.cern.ch:18081" try: api = dlsClient.getDlsApi(dls_type=type,dls_endpoint=server) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit() ## get FileBlocks given a location se="cmsboce.bo.infn.it" try: entryList=api.getFileBlocks(se) except dlsApi.DlsApiError, inst: msg = "Error in the DLS query: %s." % str(inst) print msg sys.exit() for entry in entryList: print entry.fileBlock.name
fbA = "/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#84bbc9b5-1185-4d6a-8607-ecdd93e29417" fbB = "/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#82801173-7a7b-4185-bd94-e1c71d762db5" fbC = "/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#85fa9514-b9d5-4b15-a366-9567999c31e3" seA = "srm.ciemat.es" seB = "storm.ifca.es" # ############################# ## API # ############################# print "" print " DLS Server type: %s endpoint: %s" % (type, endpoint) print "" try: api = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint, check_endpoint=True) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg rc = inst.rc sys.exit(rc) # ############################# ## List a DLS entry # ############################# # first entry fileblockA = DlsFileBlock(fbA) fileblockB = DlsFileBlock(fbB) print "*** list a DLS entry with fileblock=%s" % (fileblockA.name) try:
# // # // Local DLS instance to start from #// localDLSendpoint = MapDBSDLSinstance[localDBS] localDLStype = 'DLS_TYPE_LFC' if localDBS == 'MCLocal_1': localDLStype = 'DLS_TYPE_MYSQL' # // # // Local and Global DLS API #// print "" print " From Local DLS Server endpoint: %s (type: %s) " % (localDLSendpoint, localDLStype) try: localDLSapi = dlsClient.getDlsApi(dls_type=localDLStype, dls_endpoint=localDLSendpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit() print " to Global DLS Server endpoint: %s (type: %s)" % (endpoint, type) print "" try: globalDLSapi = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit()
def testA_CreateAPI(self): print "\n\n########## Testing API creation" api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT, check_endpoint = True) self.assert_(True)
# ############################# ## API # ############################# ## MySQL proto #type='DLS_TYPE_MSQL' #endpoint='lxgate10.cern.ch:18081' ## LFC proto #type='DLS_TYPE_LFC' # or type='DLS_TYPE_DLI' #endpoint=lfc-cms-test.cern.ch/grid/cms/fanfani/DLS/ print "" print " DLS Server type: %s endpoint: %s" % (type, endpoint) print "" try: api = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit() # ############################# ## add a DLS entry # ############################# # first entry fbA = "testblockA-part1/testblockA-part2" seA = "test-A-SE" fileblockA = DlsFileBlock(fbA) locationA = DlsLocation(seA) print "*** add a DLS entry with fileblock=%s location=%s" % (fileblockA.name, locationA.host)
print "\n >>> Upload existing block in DBS : %s "%(dbinstance) # // # // Get API to DBS #// ## database instance args = {'instance' : dbinstance} dbsapi = dbsCgiApi.DbsCgiApi(url, args) # // # // Local and Global DLS API #// print ">>>> From DLS Server endpoint: %s (type: %s) "%(inputdlsendpoint,inputdlstype) try: inDLSapi = dlsClient.getDlsApi(dls_type=inputdlstype,dls_endpoint=inputdlsendpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit() print ">>>> to DLS Server endpoint: %s (type: %s)"%(outputdlsendpoint,outputdlstype) print "" try: outDLSapi = dlsClient.getDlsApi(dls_type=outputdlstype,dls_endpoint=outputdlsendpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit()
print ">>>>> DBS URL : %s DBS Address : %s" % (url, dbinstance) print ">>>>> DLS instance : %s" % dlsendpoint # // # // Get API to DBS #// ## database instance args = {'instance': dbinstance} dbsapi = dbsCgiApi.DbsCgiApi(url, args) # // # // Get API to DLS #// try: dlsapi = dlsClient.getDlsApi(dls_type=dlstype, dls_endpoint=dlsendpoint) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg sys.exit(1) # // # // Get list of datasets #// try: if datasetPath: datasets = dbsapi.listProcessedDatasets(datasetPath) else: datasets = dbsapi.listProcessedDatasets("/*/*/*") except dbsCgiApi.DbsCgiToolError, ex: print "%s: %s " % (ex.getClassName(), ex.getErrorMessage())
fbPattern='/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#8*' fbA="/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#84bbc9b5-1185-4d6a-8607-ecdd93e29417" fbB="/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#82801173-7a7b-4185-bd94-e1c71d762db5" fbC="/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#85fa9514-b9d5-4b15-a366-9567999c31e3" seA = "srm.ciemat.es" seB = "storm.ifca.es" # ############################# ## API # ############################# print "" print " DLS Server type: %s endpoint: %s"%(type,endpoint) print "" try: api = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint, check_endpoint = True) except dlsApi.DlsApiError, inst: msg = "Error when binding the DLS interface: " + str(inst) print msg rc = inst.rc sys.exit(rc) # ############################# ## List a DLS entry # ############################# # first entry fileblockA=DlsFileBlock(fbA) fileblockB = DlsFileBlock(fbB) print "*** list a DLS entry with fileblock=%s"%(fileblockA.name) try: