def getFilesFomDAS(self, dsetName):
"""
Read dataset files from DAS.
@dsetName: dataset name
"""
## response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509())
response = das_query(
"https://cmsweb.cern.ch",
"file dataset=%s instance=%s | grep file.name,file.nevents" %
(dsetName, self.dbs_instance_),
0,
0,
False,
)
files = []
for d in response["data"]:
for jf in d["file"]:
if "nevents" in jf:
files.append({
"name": jf["name"],
"nevents": jf["nevents"]
})
break
return files
def getFilesFomDAS(self,dsetName):
"""
Read dataset files from DAS.
@dsetName: dataset name
"""
## response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509())
response = das_query("https://cmsweb.cern.ch","file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName,self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509())
files=[]
for d in response["data"]:
for jf in d["file"]:
if "nevents" in jf:
files.append({ "name" : jf["name"], "nevents" : jf["nevents"] })
break
return files
def importFromDAS(self, list_datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
#
ret, out = commands.getstatusoutput("voms-proxy-info -e")
if ret != 0:
print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n"
sys.exit(-1)
catalog = self.readCatalog()
print "Importing from das %s" % list_datasets
datasets = []
for dataset in list_datasets:
if "*" in dataset:
# response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509())
# response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509())
response = das_query(
"https://cmsweb.cern.ch",
"dataset dataset=%s instance=%s | grep dataset.name" %
(dataset, self.dbs_instance_),
0,
0,
False,
ckey=x509(),
cert=x509())
## print response
for d in response["data"]:
## print d
datasets.append(d["dataset"][0]["name"])
else:
datasets.append(dataset)
print "Datasets to import"
print "\n".join(datasets)
for dsetName in datasets:
print "Importing %s" % dsetName
files = self.getFilesFomDAS(dsetName)
self.addToDataset(catalog, dsetName, files)
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
def importFromDAS(self, list_datasets):
"""
Import datasets from DAS to the catalog.
@datasets: wildecard to be usd in dataset query
"""
#
ret, out = commands.getstatusoutput("voms-proxy-info -e")
if ret != 0:
print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n"
sys.exit(-1)
catalog = self.readCatalog()
print "Importing from das %s" % list_datasets
datasets = []
for dataset in list_datasets:
if "*" in dataset:
# response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509())
# response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509())
response = das_query(
"https://cmsweb.cern.ch",
"dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_),
0,
0,
False,
)
## print response
for d in response["data"]:
## print d
datasets.append(d["dataset"][0]["name"])
else:
datasets.append(dataset)
print "Datasets to import"
print "\n".join(datasets)
for dsetName in datasets:
print "Importing %s" % dsetName
files = self.getFilesFomDAS(dsetName)
self.addToDataset(catalog, dsetName, files)
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
