def getFilesFomDAS(self, dsetName): """ Read dataset files from DAS. @dsetName: dataset name """ ## response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) response = das_query( "https://cmsweb.cern.ch", "file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName, self.dbs_instance_), 0, 0, False, ) files = [] for d in response["data"]: for jf in d["file"]: if "nevents" in jf: files.append({ "name": jf["name"], "nevents": jf["nevents"] }) break return files
def getFilesFomDAS(self,dsetName): """ Read dataset files from DAS. @dsetName: dataset name """ ## response = das_query("https://cmsweb.cern.ch","file dataset=%s | grep file.name,file.nevents" % dsetName, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) response = das_query("https://cmsweb.cern.ch","file dataset=%s instance=%s | grep file.name,file.nevents" % (dsetName,self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) files=[] for d in response["data"]: for jf in d["file"]: if "nevents" in jf: files.append({ "name" : jf["name"], "nevents" : jf["nevents"] }) break return files
def importFromDAS(self, list_datasets): """ Import datasets from DAS to the catalog. @datasets: wildecard to be usd in dataset query """ # ret, out = commands.getstatusoutput("voms-proxy-info -e") if ret != 0: print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n" sys.exit(-1) catalog = self.readCatalog() print "Importing from das %s" % list_datasets datasets = [] for dataset in list_datasets: if "*" in dataset: # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) response = das_query( "https://cmsweb.cern.ch", "dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) ## print response for d in response["data"]: ## print d datasets.append(d["dataset"][0]["name"]) else: datasets.append(dataset) print "Datasets to import" print "\n".join(datasets) for dsetName in datasets: print "Importing %s" % dsetName files = self.getFilesFomDAS(dsetName) self.addToDataset(catalog, dsetName, files) print "Writing catalog" self.writeCatalog(catalog) print "Done"
def importFromDAS(self, list_datasets): """ Import datasets from DAS to the catalog. @datasets: wildecard to be usd in dataset query """ # ret, out = commands.getstatusoutput("voms-proxy-info -e") if ret != 0: print "\n\nNo valid voms proxy found. This is needed to query DAS.\nPlease create a valid proxy running the following command:\nvoms-proxy-init -voms cms\n" sys.exit(-1) catalog = self.readCatalog() print "Importing from das %s" % list_datasets datasets = [] for dataset in list_datasets: if "*" in dataset: # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s | grep dataset.name" % dataset, 0, 0, False, self.dbs_instance_, ckey=x509(), cert=x509()) # response = das_query("https://cmsweb.cern.ch","dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ckey=x509(), cert=x509()) response = das_query( "https://cmsweb.cern.ch", "dataset dataset=%s instance=%s | grep dataset.name" % (dataset, self.dbs_instance_), 0, 0, False, ) ## print response for d in response["data"]: ## print d datasets.append(d["dataset"][0]["name"]) else: datasets.append(dataset) print "Datasets to import" print "\n".join(datasets) for dsetName in datasets: print "Importing %s" % dsetName files = self.getFilesFomDAS(dsetName) self.addToDataset(catalog, dsetName, files) print "Writing catalog" self.writeCatalog(catalog) print "Done"