def create_iface_binding(iface_type, endpoint, dbsConf, verbose):
if iface_type == dlsClient.DLS_TYPE_PHEDEX:
iface = dlsClient.getDlsApi(iface_type, endpoint, dbs_client_config=dbsConf,
uaFlexString = 'DLS-CLI')
else:
iface = dlsClient.getDlsApi(iface_type, endpoint, dbs_client_config=dbsConf)
if(verbose == 2):
iface.setVerbosity(DLS_VERB_HIGH)
else:
if(verbose == 0):
iface.setVerbosity(DLS_VERB_NONE)
else:
if(verbose == 1):
iface.setVerbosity(DLS_VERB_WARN)
return iface
def __init__(self, type, cfg_params):
self.cfg_params = cfg_params
self.showCAF = False
self.showProd = False
self.subscribed = False
subscribed = int(self.cfg_params.get('CMSSW.subscribed', 0))
if subscribed == 1: self.subscribed = True
showProd = int(self.cfg_params.get('CMSSW.show_prod', 0))
if showProd == 1: self.showProd = True
phedexURL = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
global_url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
if type == "DLS_TYPE_DBS":
# use dbs_url as dls_endpoint if dls_type is dbs
endpoint = self.cfg_params.get('CMSSW.dbs_url', global_url)
elif type == "DLS_TYPE_PHEDEX":
endpoint = self.cfg_params.get('CMSSW.dls_phedex_url', phedexURL)
if self.cfg_params['CRAB.scheduler'].upper() == 'CAF':
self.showCAF = True
else:
msg = "DLS type %s not among the supported DLS ( DLS_TYPE_DLI and DLS_TYPE_MYSQL ) " % type
raise CrabException(msg)
common.logger.debug("DLS interface: %s Server %s" % (type, endpoint))
try:
self.api = dlsClient.getDlsApi(dls_type=type,
dls_endpoint=endpoint,
verbosity=dlsApi.DLS_VERB_INFO)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: %s Server %s" % (
str(inst), endpoint)
#print msg
raise CrabException(msg)
class DBS3Reader:
"""
_DBSReader_
General API for reading data from DBS
"""
def __init__(self, url, **contact):
# instantiate dbs api object
try:
self.dbs = DbsApi(url, **contact)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
# setup DLS api
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl,
version='')
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR', "version": ''}
args.update(contact)
#try:
self.dbs = DbsApi(args)
#except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl,
version=args['version'])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
def __init__(self, type, cfg_params):
self.cfg_params = cfg_params
self.showCAF = False
self.showProd = False
self.subscribed = False
subscribed =int(self.cfg_params.get('CMSSW.subscribed', 0))
if subscribed == 1: self.subscribed = True
showProd = int(self.cfg_params.get('CMSSW.show_prod', 0))
if showProd == 1: self.showProd = True
phedexURL='https://cmsweb.cern.ch/phedex/datasvc/xml/prod/'
global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
if type=="DLS_TYPE_DBS":
# use dbs_url as dls_endpoint if dls_type is dbs
endpoint=self.cfg_params.get('CMSSW.dbs_url', global_url)
elif type=="DLS_TYPE_PHEDEX":
endpoint=self.cfg_params.get('CMSSW.dls_phedex_url',phedexURL)
if self.cfg_params['CRAB.scheduler'].upper() == 'CAF': self.showCAF = True
else:
msg = "DLS type %s not among the supported DLS ( DLS_TYPE_DLI and DLS_TYPE_MYSQL ) "%type
raise CrabException(msg)
common.logger.debug("DLS interface: %s Server %s"%(type,endpoint))
try:
self.api = dlsClient.getDlsApi(dls_type=type,dls_endpoint=endpoint,verbosity=dlsApi.DLS_VERB_INFO)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: %s Server %s"%(str(inst),endpoint)
#print msg
raise CrabException(msg)
def testF_GetBlocksOfLocationList(self):
print "\n\n########## Testing the retrieval of fileblocks for several locations"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Get FileBlocks given the locations: %s, %s" % (seA, seB)
entryList = api.getFileBlocks([self.C.locA, self.C.locB], errorTolerant=False)
fList = [x.fileBlock.name for x in entryList]
print "\nlen(fList): %s" % len(fList)
self.assert_(len(fList) > self.C.lenListSeA)
def testB_ListOneFileblock(self):
print "\n\n########## Testing the listing of one fileblock"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "List a DLS entry with fileblock=%s" % fbA
fList = api.listFileBlocks(self.C.fblockA)
print "\nfList: %s" % ([x.name for x in fList],)
self.assert_(len(fList) == 1)
self.assertEqual(fList[0].name, fbA)
def __init__(self, dls_type, dls_address):
self.conf = {}
self.conf['type'] = dls_type
self.conf['address'] = dls_address
try:
logging.debug('Instantiating DLS client interface')
self.api= dlsClient.getDlsApi(dls_type = dls_type,\
dls_endpoint = dls_address)
logging.debug('DLS client interface instantiated')
except dlsApi.DlsApiError, inst:
raise ProdException(exceptions[4010] + str(inst), 4010)
def __init__(self,dls_type,dls_address):
self.conf={}
self.conf['type']=dls_type
self.conf['address']=dls_address
try:
logging.debug('Instantiating DLS client interface')
self.api= dlsClient.getDlsApi(dls_type = dls_type,\
dls_endpoint = dls_address)
logging.debug('DLS client interface instantiated')
except dlsApi.DlsApiError, inst:
raise ProdException(exceptions[4010]+str(inst),4010)
def testC_ListFileblockPattern(self):
print "\n\n########## Testing the listing of one fileblock and a pattern"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "List several DLS entries for pattern=%s and block=%s"%(fbPattern, fbB)
fList = api.listFileBlocks([self.C.fblockPat, self.C.fblockB])
print "\nfList: %s" % ([x.name for x in fList],)
self.assert_(len(fList) >= 2)
self.assert_(fbA in [x.name for x in fList])
self.assert_(fbB in [x.name for x in fList])
# Store fileblock list for use at test H
self.C.fListFbPat = fList
def testJ_GetAllLocations(self):
print "\n\n########## Getting all existing locations"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
locList = api.getAllLocations()
self.assert_(len(locList) > 20)
locs = [x.host for x in locList]
print "All locations: %s" % locs
self.assert_(seA in locs)
self.assert_(seB in locs)
for loc in self.C.locsFblockSeA:
self.assert_(loc in locs)
def testG_GetLocationOfOneBlockWithT01(self):
print "\n\n########## Getting location of one fileblock including T0 and T1s"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Get location of fileblock=%s" % self.C.fblockSeA
entryList = api.getLocations(self.C.fblockSeA, errorTolerant=False, showProd=True)
self.assert_(len(entryList) == 1)
locs = [x.host for x in entryList[0].locations]
print "\nlocations: %s" % locs
self.assert_(seA in locs)
self.assert_(len(locs) > len(self.C.locsFblockSeA))
for loc in self.C.locsFblockSeA:
self.assert_(loc in locs)
def testE_GetLocationOfOneBlock(self):
print "\n\n########## Getting location of one fileblock"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Get location of fileblock=%s" % self.C.fblockSeA
entryList = api.getLocations(self.C.fblockSeA, errorTolerant = False)
self.assert_(len(entryList) == 1)
locs = [x.host for x in entryList[0].locations]
print "\nlocations: %s" % locs
self.assert_(seA in locs)
# Store the locations list to compare with those of tests G and I
self.C.locsFblockSeA = locs
def testD_GetBlocksOfLocation(self):
print "\n\n########## Testing the retrieval of fileblocks for one location"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Get FileBlocks given the location: %s" % (seA)
entryList = api.getFileBlocks(self.C.locA, errorTolerant=False)
fList = [x.fileBlock.name for x in entryList]
print "\nlen(fList): %s" % len(fList)
self.assert_(len(fList) > 1)
# Store one fileblock at the given location for next test
self.C.fblockSeA = fList[0]
# Store the block list length to compare with test F
self.C.lenListSeA = len(fList)
def testL_QueryWithUserAgent(self):
print "\n\n########## Querying with user agent set"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT,
uaFlexString = 'This is a test',
uaClientsList = [['Tester', '1.1'], ['Sth', '2_4']])
print "Get location of fileblock=%s" % self.C.fblockSeA
entryList = api.getLocations(self.C.fblockSeA, errorTolerant = False)
self.assert_(len(entryList) == 1)
locs = [x.host for x in entryList[0].locations]
print "\nlocations: %s" % locs
self.assert_(seA in locs)
# Store the locations list to compare with those of tests G and I
self.C.locsFblockSeA = locs
def testK_GetFileLocsForOneBlock(self):
print "\n\n########## Getting file locs for one fileblock"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Get file locs for: %s" % self.C.fblockSeA
fileList = api.getFileLocs(self.C.fblockSeA)
self.assert_(len(fileList) == 1)
block = fileList[0][0]
self.assertEqual(block.name, self.C.fblockSeA)
fileLocs = fileList[0][1]
self.assert_(len(fileLocs) > 1)
for f in fileLocs:
for loc in fileLocs[f]:
# print 'f, locs: %s, %s' % (f, fileLocs[f])
self.assert_(loc.host in self.C.locsFblockSeA)
def testI_GetLocationOfListOfBlocks(self):
print "\n\n########## Getting locations of a list of fileblocks"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Get location of fileblocks: %s, %s" % (self.C.fblockSeA, fbA)
entryList = api.getLocations([self.C.fblockSeA, self.C.fblockA], errorTolerant=False)
for entry in entryList:
block = entry.fileBlock.name
locs = [x.host for x in entry.locations]
print "\nlocations for block %s: %s" % (block, locs)
if block == fbA:
self.assert_(len(entry.locations) > 0)
else:
self.assertEqual(len(entry.locations), len(self.C.locsFblockSeA))
for loc in self.C.locsFblockSeA:
self.assert_(loc in locs)
def testH_DumpEntriesWithPattern(self):
print "\n\n########## Dump several entries for pattern"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT)
print "Dumping entries for pattern: %s" % fbPattern
entryList = api.dumpEntries(fbPattern, errorTolerant = False)
self.assert_(len(entryList) > 1)
fList = [x.fileBlock.name for x in entryList]
print "\nfList: %s" % fList
print "\nself.C.fListFbPat: %s" % [x.name for x in self.C.fListFbPat]
# Check the fileblocks are the same as those listed before
self.assertEqual(len(fList), len(self.C.fListFbPat))
for fb in fList:
self.assert_(fb in [x.name for x in self.C.fListFbPat])
# Check each fileblock has a non-emtpy list of locations
for entry in entryList:
self.assert_(len(entry.locations) > 0)
def __init__(self, url, **contact):
args = {"url": url, "level": "ERROR", "version": ""}
args.update(contact)
# try:
self.dbs = DbsApi(args)
# except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count("cmsdbsprod.cern.ch/cms_dbs_prod_global") or self.dbs.getServerInfo()["InstanceName"] == "GLOBAL":
dlsType = "DLS_TYPE_PHEDEX"
dlsUrl = "https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
else:
dlsType = "DLS_TYPE_DBS"
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version=args["version"])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
hostname = 'localhost'
pass
msg ="At %s found %s %s processes on %s.\n"%(time.asctime(),nProc,procName,hostname)
msg+="Please take some action, since it indicates that DLS is not responding.\n"
msg+="The %s is a part of DBS data discovery and runs as cronjob on %s."%(procName,hostname)
DBSUtil.sendEmail(msg)
sys.exit(1)
# get grid proxy
# cmd="cat $HOME/.globus/pp.txt | grid-proxy-init -pwstdin -q"
cmd="cat /data/DBSDataDiscovery/COMP/DBS/Web/DataDiscovery/pp.txt | grid-proxy-init -cert /data/vk/cert/usercert.pem -key /data/vk/cert/userkey.pem -pwstdin -q"
os.system(cmd)
file = open('dls.all.tmp','w')
for dbsInst in DBSInst.DBS_DLS_INST.keys():
iface=""
try:
url,dlsType,endpoint = DBSInst.DBS_DLS_INST[dbsInst]
if dlsType=='DLS_TYPE_DLI':
dlsType='DLS_TYPE_LFC'
iface = dlsClient.getDlsApi(dlsType,endpoint)
#print dbsInst,dlsType,endpoint,iface
locList = iface.getAllLocations(session = True)
for loc in locList:
file.write('%s %s\n'%(dbsInst,loc.host))
#DBSUtil.printListElements(lList,dbsInst)
except:
printExcept()
raise "Fail to generate location list for dbs instance",dbsInst
file.close()
os.rename('dls.all.tmp','dls.all')
# //
# // Local DLS instance to start from
#//
localDLSendpoint=MapDBSDLSinstance[localDBS]
localDLStype='DLS_TYPE_LFC'
if localDBS == 'MCLocal_1':
localDLStype='DLS_TYPE_MYSQL'
# //
# // Local and Global DLS API
#//
print ""
print " From Local DLS Server endpoint: %s (type: %s) "%(localDLSendpoint,localDLStype)
try:
localDLSapi = dlsClient.getDlsApi(dls_type=localDLStype,dls_endpoint=localDLSendpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
print " to Global DLS Server endpoint: %s (type: %s)"%(endpoint,type)
print ""
try:
globalDLSapi = dlsClient.getDlsApi(dls_type=type,dls_endpoint=endpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
return msg
##################################################333
# Unit testing
if __name__ == "__main__":
import dlsClient
from dlsDataObjects import *
## use DLS server
type="DLS_TYPE_MYSQL"
server ="lxgate10.cern.ch:18081"
try:
api = dlsClient.getDlsApi(dls_type=type,dls_endpoint=server)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
## get FileBlocks given a location
se="cmsboce.bo.infn.it"
try:
entryList=api.getFileBlocks(se)
except dlsApi.DlsApiError, inst:
msg = "Error in the DLS query: %s." % str(inst)
print msg
sys.exit()
for entry in entryList:
print entry.fileBlock.name
fbA = "/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#84bbc9b5-1185-4d6a-8607-ecdd93e29417"
fbB = "/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#82801173-7a7b-4185-bd94-e1c71d762db5"
fbC = "/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#85fa9514-b9d5-4b15-a366-9567999c31e3"
seA = "srm.ciemat.es"
seB = "storm.ifca.es"
# #############################
## API
# #############################
print ""
print " DLS Server type: %s endpoint: %s" % (type, endpoint)
print ""
try:
api = dlsClient.getDlsApi(dls_type=type,
dls_endpoint=endpoint,
check_endpoint=True)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
rc = inst.rc
sys.exit(rc)
# #############################
## List a DLS entry
# #############################
# first entry
fileblockA = DlsFileBlock(fbA)
fileblockB = DlsFileBlock(fbB)
print "*** list a DLS entry with fileblock=%s" % (fileblockA.name)
try:
# //
# // Local DLS instance to start from
#//
localDLSendpoint = MapDBSDLSinstance[localDBS]
localDLStype = 'DLS_TYPE_LFC'
if localDBS == 'MCLocal_1':
localDLStype = 'DLS_TYPE_MYSQL'
# //
# // Local and Global DLS API
#//
print ""
print " From Local DLS Server endpoint: %s (type: %s) " % (localDLSendpoint,
localDLStype)
try:
localDLSapi = dlsClient.getDlsApi(dls_type=localDLStype,
dls_endpoint=localDLSendpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
print " to Global DLS Server endpoint: %s (type: %s)" % (endpoint, type)
print ""
try:
globalDLSapi = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
def testA_CreateAPI(self):
print "\n\n########## Testing API creation"
api = dlsClient.getDlsApi(dls_type=TYPE, dls_endpoint=ENDPOINT, check_endpoint = True)
self.assert_(True)
# #############################
## API
# #############################
## MySQL proto
#type='DLS_TYPE_MSQL'
#endpoint='lxgate10.cern.ch:18081'
## LFC proto
#type='DLS_TYPE_LFC' # or type='DLS_TYPE_DLI'
#endpoint=lfc-cms-test.cern.ch/grid/cms/fanfani/DLS/
print ""
print " DLS Server type: %s endpoint: %s" % (type, endpoint)
print ""
try:
api = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
# #############################
## add a DLS entry
# #############################
# first entry
fbA = "testblockA-part1/testblockA-part2"
seA = "test-A-SE"
fileblockA = DlsFileBlock(fbA)
locationA = DlsLocation(seA)
print "*** add a DLS entry with fileblock=%s location=%s" % (fileblockA.name,
locationA.host)
print "\n >>> Upload existing block in DBS : %s "%(dbinstance)
# //
# // Get API to DBS
#//
## database instance
args = {'instance' : dbinstance}
dbsapi = dbsCgiApi.DbsCgiApi(url, args)
# //
# // Local and Global DLS API
#//
print ">>>> From DLS Server endpoint: %s (type: %s) "%(inputdlsendpoint,inputdlstype)
try:
inDLSapi = dlsClient.getDlsApi(dls_type=inputdlstype,dls_endpoint=inputdlsendpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
print ">>>> to DLS Server endpoint: %s (type: %s)"%(outputdlsendpoint,outputdlstype)
print ""
try:
outDLSapi = dlsClient.getDlsApi(dls_type=outputdlstype,dls_endpoint=outputdlsendpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit()
print ">>>>> DBS URL : %s DBS Address : %s" % (url, dbinstance)
print ">>>>> DLS instance : %s" % dlsendpoint
# //
# // Get API to DBS
#//
## database instance
args = {'instance': dbinstance}
dbsapi = dbsCgiApi.DbsCgiApi(url, args)
# //
# // Get API to DLS
#//
try:
dlsapi = dlsClient.getDlsApi(dls_type=dlstype, dls_endpoint=dlsendpoint)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
sys.exit(1)
# //
# // Get list of datasets
#//
try:
if datasetPath:
datasets = dbsapi.listProcessedDatasets(datasetPath)
else:
datasets = dbsapi.listProcessedDatasets("/*/*/*")
except dbsCgiApi.DbsCgiToolError, ex:
print "%s: %s " % (ex.getClassName(), ex.getErrorMessage())
fbPattern='/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#8*'
fbA="/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#84bbc9b5-1185-4d6a-8607-ecdd93e29417"
fbB="/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#82801173-7a7b-4185-bd94-e1c71d762db5"
fbC="/Zmumu/Summer09-MC_31X_V3-v1/GEN-SIM-RECO#85fa9514-b9d5-4b15-a366-9567999c31e3"
seA = "srm.ciemat.es"
seB = "storm.ifca.es"
# #############################
## API
# #############################
print ""
print " DLS Server type: %s endpoint: %s"%(type,endpoint)
print ""
try:
api = dlsClient.getDlsApi(dls_type=type, dls_endpoint=endpoint, check_endpoint = True)
except dlsApi.DlsApiError, inst:
msg = "Error when binding the DLS interface: " + str(inst)
print msg
rc = inst.rc
sys.exit(rc)
# #############################
## List a DLS entry
# #############################
# first entry
fileblockA=DlsFileBlock(fbA)
fileblockB = DlsFileBlock(fbB)
print "*** list a DLS entry with fileblock=%s"%(fileblockA.name)
try:
