def prepareDeletionRequests(datasets):
"""
make a single deletion request per bunch of datasets
Filtering only the INVALID or DEPRECATED ones
"""
size = 30
#delete duplicates
datasets = list(set(datasets))
#group datasets by sites
requests = {}
for ds in datasets:
try:
t = dbs.getDatasetStatus(ds)
#filter by status
if t != 'INVALID' and t != 'DEPRECATED':
continue
sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
for s in sites:
#ignore buffers
if "Buffer" in s or "Export" in s:
continue
if s not in requests:
requests[s] = []
requests[s].append(ds)
except Exception as e:
print "error:",ds,e
return requests
def prepareDeletionRequests(datasets):
"""
make a single deletion request per bunch of datasets
Filtering only the INVALID or DEPRECATED ones
"""
size = 30
#delete duplicates
datasets = list(set(datasets))
#group datasets by sites
requests = {}
for ds in datasets:
try:
t = dbs.getDatasetStatus(ds)
#filter by status
if t != 'INVALID' and t != 'DEPRECATED':
continue
sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
for s in sites:
#ignore buffers
if "Buffer" in s or "Export" in s:
continue
if s not in requests:
requests[s] = []
requests[s].append(ds)
except Exception as e:
print "error:", ds, e
return requests
def makeDeletionRequests(url, allDatasets, verbose=False):
"""
make a single deletion request per bunch of datasets
Filtering only the INVALID or DEPRECATED ones
"""
size = 20
deletionRequests = []
#delete duplicates
allDatasets = list(set(allDatasets))
while allDatasets:
datasets = allDatasets[:size]
allDatasets = allDatasets[size:]
#get the sites
sites = set()
#add all sites for all datasets
dsToDelete = set()
for ds in datasets:
try:
t = dbs.getDatasetStatus(ds)
if verbose:
print ds, 'is', t
#filter by status
if t == 'INVALID' or t == 'DEPRECATED':
dsToDelete.add(ds)
sites2 = phd.getBlockReplicaSites(ds, onlycomplete=False)
for s in sites2:
#ignore buffers
if "Buffer" in s or "Export" in s:
continue
sites.add(s)
if verbose:
print "available in", sites
except Exception as e:
print ds,e
#create a single request
if dsToDelete and sites:
print "About to create a deletion request for"
print '\n'.join(dsToDelete)
print "To this sites:"
print '\n'.join(sites)
r = phd.makeDeletionRequest(url, list(sites), dsToDelete, "Invalid data, can be deleted")
if ("phedex" in r and
"request_created" in r["phedex"] and
"id" in r["phedex"]["request_created"]):
reqid = r["phedex"]["request_created"]["id"]
deletionRequests.append(reqid)
if verbose:
print "Request created:", reqid
else:
print r
return deletionRequests
def makeDeletionRequests(url, allDatasets, verbose=False, test=False):
"""
make a single deletion request per bunch of datasets
Filtering only the INVALID or DEPRECATED ones
"""
size = 30
#delete duplicates
datasets = list(set(allDatasets))
#group datasets by sites
requests = {}
for ds in datasets:
try:
t = dbs.getDatasetStatus(ds)
if verbose:
print ds, 'is', t
#filter by status
if t != 'INVALID' and t != 'DEPRECATED':
continue
sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
for s in sites:
#ignore buffers
if "Buffer" in s or "Export" in s:
continue
if s not in requests:
requests[s] = []
requests[s].append(ds)
if verbose:
print "available in", sites
except Exception as e:
print ds,e
deletionRequests = []
#for each site
for s in sorted(requests.keys()):
datasets = requests[s]
print "site", s
print "datasets to delete"
print '\n'.join(datasets)
if not test:
r = phd.makeDeletionRequest(url, [s], datasets, "Invalid data, can be deleted")
if ("phedex" in r and
"request_created" in r["phedex"]):
reqid = r["phedex"]["request_created"][0]["id"]
deletionRequests.append(reqid)
if verbose:
print "Request created:", reqid
else:
print r
return deletionRequests
def makeDeletionRequests(url, allDatasets, verbose=False, test=False):
"""
make a single deletion request per bunch of datasets
Filtering only the INVALID or DEPRECATED ones
"""
size = 30
#delete duplicates
datasets = list(set(allDatasets))
#group datasets by sites
requests = {}
for ds in datasets:
try:
t = dbs.getDatasetStatus(ds)
if verbose:
print ds, 'is', t
#filter by status
if t != 'INVALID' and t != 'DEPRECATED':
continue
sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
for s in sites:
#ignore buffers
if "Buffer" in s or "Export" in s:
continue
if s not in requests:
requests[s] = []
requests[s].append(ds)
if verbose:
print "available in", sites
except Exception as e:
print ds, e
deletionRequests = []
#for each site
for s in sorted(requests.keys()):
datasets = requests[s]
print "site", s
print "datasets to delete"
print '\n'.join(datasets)
if not test:
r = phd.makeDeletionRequest(url, [s], datasets,
"Invalid data, can be deleted")
if ("phedex" in r and "request_created" in r["phedex"]):
reqid = r["phedex"]["request_created"][0]["id"]
deletionRequests.append(reqid)
if verbose:
print "Request created:", reqid
else:
print r
return deletionRequests
def main():
parser = optparse.OptionParser("pyton %prog [DSET1 DSET2 ... | -f FILE]")
parser.add_option('-f', '--file', help='Text file', dest='file')
(options, args) = parser.parse_args()
if options.file:
datasets = [l.strip() for l in open(options.file) if l.strip()]
elif len(args) >= 1:
datasets = args
else:
parser.error("A file name or dataset is required")
sys.exit(0)
for dataset in datasets:
status = dbs3.getDatasetStatus(dataset)
print dataset, status
def main():
url='cmsweb.cern.ch'
parser = optparse.OptionParser()
parser.add_option('-f', '--filename', help='Filename',dest='filename')
parser.add_option('-d', '--dataset', help='Dataset',dest='userDataset')
(options,args) = parser.parse_args()
if not options.filename and not options.userDataset:
print "A filename or dataset is required"
sys.exit(0)
filename=options.filename
if options.filename:
f=open(filename,'r')
else:
f=[options.userDataset]
for dataset in f:
dataset = dataset.rstrip('\n')
status = dbs3.getDatasetStatus(dataset)
print dataset, status
sys.exit(0)
def main():
url = 'cmsweb.cern.ch'
parser = optparse.OptionParser()
parser.add_option('-f', '--filename', help='Filename', dest='filename')
parser.add_option('-d', '--dataset', help='Dataset', dest='userDataset')
(options, args) = parser.parse_args()
if not options.filename and not options.userDataset:
print "A filename or dataset is required"
sys.exit(0)
filename = options.filename
if options.filename:
f = open(filename, 'r')
else:
f = [options.userDataset]
for dataset in f:
dataset = dataset.rstrip('\n')
status = dbs3.getDatasetStatus(dataset)
print dataset, status
sys.exit(0)
