def importFromDAS(self,datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
catalog = self.readCatalog()
print "Importing from das %s" % datasets
if "*" in datasets:
response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % datasets, 0, 0, False, self.dbs_instance_)
datasets=[]
for d in response["data"]:
datasets.append( d["dataset"][0]["name"] )
print "Datasets to import"
print "\n".join(datasets)
for dsetName in datasets:
print "Importing %s" % dsetName
files = self.getFilesFomDAS(dsetName)
if dsetName in catalog:
catalog[ dsetName ]["files"] = files
else:
catalog[ dsetName ] = { "files" : files }
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
def importFromDAS(self,list_datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
catalog = self.readCatalog()
print "Importing from das %s" % list_datasets
datasets = []
for dataset in list_datasets:
if "*" in dataset:
response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_)
for d in response["data"]:
datasets.append( d["dataset"][0]["name"] )
else:
datasets.append(dataset)
print "Datasets to import"
print "\n".join(datasets)
for dsetName in datasets:
print "Importing %s" % dsetName
files = self.getFilesFomDAS(dsetName)
self.addToDataset(catalog,dsetName,files)
## if dsetName in catalog:
## if self.force_:
## catalog[ dsetName ]["files"] = files
## else:
## self.mergeDataset(catalog[ dsetName ],{ "files" : files })
## else:
## catalog[ dsetName ] = { "files" : files }
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
def importFromDAS(self,list_datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
#
ret,out = commands.getstatusoutput("voms-proxy-info -e")
if ret != 0:
print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n"
sys.exit(-1)
catalog = self.readCatalog()
print "Importing from das %s" % list_datasets
datasets = []
for dataset in list_datasets:
if "*" in dataset:
response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509())
## print response
for d in response["data"]:
## print d
datasets.append( d["dataset"][0]["name"] )
else:
datasets.append(dataset)
print "Datasets to import"
print "\n".join(datasets)
for dsetName in datasets:
print "Importing %s" % dsetName
files = self.getFilesFomDAS(dsetName)
self.addToDataset(catalog,dsetName,files)
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
def importFromDAS(self,list_datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
catalog = self.readCatalog()
print "Importing from das %s" % list_datasets
datasets = []
for dataset in list_datasets:
if "*" in dataset:
response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_)
for d in response["data"]:
datasets.append( d["dataset"][0]["name"] )
else:
datasets.append(dataset)
print "Datasets to import"
print "\n".join(datasets)
for dsetName in datasets:
print "Importing %s" % dsetName
files = self.getFilesFomDAS(dsetName)
if dsetName in catalog:
catalog[ dsetName ]["files"] = files
else:
catalog[ dsetName ] = { "files" : files }
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
def importFromDAS(self, datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
catalog = self.readCatalog()
if "*" in datasets:
response = das_query(
"https://cmsweb.cern.ch",
"dataset dataset=%s | grep dataset.name" % datasets, 0, 0,
False, self.dbs_instance_)
datasets = []
for d in response["data"]:
datasets.append(d["dataset"][0]["name"])
for dsetName in datasets:
files = self.getFilesFomDAS(dsetName)
if dsetName in catalog:
catalog[dsetName]["files"] = files
else:
catalog[dsetName] = {"files": files}
self.writeCatalog(catalog)
def getFilesFomDAS(self,dsetName):
"""
Read dataset files from DAS.
@dsetName: dataset name
"""
response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_)
files=[]
for d in response["data"]:
files.append( { "name" : d["file"][0]["name"], "nevents" : d["file"][0]["nevents"] } )
return files
def getFilesFomDAS(self,dsetName):
"""
Read dataset files from DAS.
@dsetName: dataset name
"""
response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509())
files=[]
for d in response["data"]:
for jf in d["file"]:
if "nevents" in jf:
files.append({ "name" : jf["name"], "nevents" : jf["nevents"] })
break
return files
def getFilesFomDAS(self,dsetName):
"""
Read dataset files from DAS.
@dsetName: dataset name
"""
response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_)
files=[]
for d in response["data"]:
for jf in d["file"]:
if "nevents" in jf:
files.append({ "name" : jf["name"], "nevents" : jf["nevents"] })
break
## files.append( { "name" : d["file"][0]["name"], "nevents" : d["file"][0]["nevents"] } )
return files
