예제 #1
0
def prepareDeletionRequests(datasets):
    """
    make a single deletion request per bunch of datasets
    Filtering only the INVALID or DEPRECATED ones
    """
    
    size = 30

    #delete duplicates
    datasets = list(set(datasets))
    
    #group datasets by sites
    requests = {}
    for ds in datasets:
        try:
            t = dbs.getDatasetStatus(ds)
            #filter by status
            if t != 'INVALID' and t != 'DEPRECATED':
                continue
            sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
            for s in sites:
                #ignore buffers
                if "Buffer" in s or "Export" in s:
                    continue
                if s not in requests:
                    requests[s] = []
                requests[s].append(ds)
        except Exception as e:
            print "error:",ds,e
    

    return requests
예제 #2
0
def prepareDeletionRequests(datasets):
    """
    make a single deletion request per bunch of datasets
    Filtering only the INVALID or DEPRECATED ones
    """

    size = 30

    #delete duplicates
    datasets = list(set(datasets))

    #group datasets by sites
    requests = {}
    for ds in datasets:
        try:
            t = dbs.getDatasetStatus(ds)
            #filter by status
            if t != 'INVALID' and t != 'DEPRECATED':
                continue
            sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
            for s in sites:
                #ignore buffers
                if "Buffer" in s or "Export" in s:
                    continue
                if s not in requests:
                    requests[s] = []
                requests[s].append(ds)
        except Exception as e:
            print "error:", ds, e

    return requests
def makeDeletionRequests(url, allDatasets, verbose=False):
    """
    make a single deletion request per bunch of datasets
    Filtering only the INVALID or DEPRECATED ones
    """
    
    size = 20
    deletionRequests = []
    #delete duplicates
    allDatasets = list(set(allDatasets))
    while allDatasets:
        datasets = allDatasets[:size]
        allDatasets = allDatasets[size:]
        
        #get the sites
        sites = set()
        #add all sites for all datasets
        dsToDelete = set()
        for ds in datasets:
            try:
                t = dbs.getDatasetStatus(ds)
                if verbose:
                    print ds, 'is', t
                #filter by status
                if t == 'INVALID' or t == 'DEPRECATED':
                    dsToDelete.add(ds)
                sites2 = phd.getBlockReplicaSites(ds, onlycomplete=False)
                for s in sites2:
                    #ignore buffers
                    if "Buffer" in s or "Export" in s:
                        continue
                    sites.add(s)
                if verbose:
                    print "available in", sites
            except Exception as e:
                print ds,e
                
        #create a single request
        if dsToDelete and sites:
            print "About to create a deletion request for"
            print '\n'.join(dsToDelete)
            print "To this sites:"
            print '\n'.join(sites)
            r = phd.makeDeletionRequest(url, list(sites), dsToDelete, "Invalid data, can be deleted")
            if ("phedex" in r and 
                    "request_created" in r["phedex"] and
                    "id" in r["phedex"]["request_created"]):
                reqid = r["phedex"]["request_created"]["id"]
                deletionRequests.append(reqid)
                if verbose:
                    print "Request created:", reqid
            else:
                print r
    return deletionRequests
def makeDeletionRequests(url, allDatasets, verbose=False, test=False):
    """
    make a single deletion request per bunch of datasets
    Filtering only the INVALID or DEPRECATED ones
    """
    
    size = 30

    #delete duplicates
    datasets = list(set(allDatasets))
    
    #group datasets by sites
    requests = {}
    for ds in datasets:
        try:
            t = dbs.getDatasetStatus(ds)
            if verbose:
                print ds, 'is', t
            #filter by status
            if t != 'INVALID' and t != 'DEPRECATED':
                continue
            sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
            for s in sites:
                #ignore buffers
                if "Buffer" in s or "Export" in s:
                    continue
                if s not in requests:
                    requests[s] = []
                requests[s].append(ds)
            if verbose:
                print "available in", sites
        except Exception as e:
            print ds,e
    

    deletionRequests = []
    #for each site
    for s in sorted(requests.keys()):
        datasets = requests[s]
        print "site", s
        print "datasets to delete"
        print '\n'.join(datasets)
        if not test:
            r = phd.makeDeletionRequest(url, [s], datasets, "Invalid data, can be deleted")
            if ("phedex" in r and 
                    "request_created" in r["phedex"]):
                reqid = r["phedex"]["request_created"][0]["id"]
                deletionRequests.append(reqid)
                if verbose:
                    print "Request created:", reqid
            else:
                print r
    return deletionRequests
def makeDeletionRequests(url, allDatasets, verbose=False, test=False):
    """
    make a single deletion request per bunch of datasets
    Filtering only the INVALID or DEPRECATED ones
    """

    size = 30

    #delete duplicates
    datasets = list(set(allDatasets))

    #group datasets by sites
    requests = {}
    for ds in datasets:
        try:
            t = dbs.getDatasetStatus(ds)
            if verbose:
                print ds, 'is', t
            #filter by status
            if t != 'INVALID' and t != 'DEPRECATED':
                continue
            sites = phd.getBlockReplicaSites(ds, onlycomplete=False)
            for s in sites:
                #ignore buffers
                if "Buffer" in s or "Export" in s:
                    continue
                if s not in requests:
                    requests[s] = []
                requests[s].append(ds)
            if verbose:
                print "available in", sites
        except Exception as e:
            print ds, e

    deletionRequests = []
    #for each site
    for s in sorted(requests.keys()):
        datasets = requests[s]
        print "site", s
        print "datasets to delete"
        print '\n'.join(datasets)
        if not test:
            r = phd.makeDeletionRequest(url, [s], datasets,
                                        "Invalid data, can be deleted")
            if ("phedex" in r and "request_created" in r["phedex"]):
                reqid = r["phedex"]["request_created"][0]["id"]
                deletionRequests.append(reqid)
                if verbose:
                    print "Request created:", reqid
            else:
                print r
    return deletionRequests
예제 #6
0
def main():
    parser = optparse.OptionParser("pyton %prog [DSET1 DSET2 ... | -f FILE]")
    parser.add_option('-f', '--file', help='Text file', dest='file')
    (options, args) = parser.parse_args()

    if options.file:
        datasets = [l.strip() for l in open(options.file) if l.strip()]
    elif len(args) >= 1:
        datasets = args
    else:
        parser.error("A file name or dataset is required")
        sys.exit(0)
    for dataset in datasets:
        status = dbs3.getDatasetStatus(dataset)
        print dataset, status
예제 #7
0
def main():
	url='cmsweb.cern.ch'	
	parser = optparse.OptionParser()
	parser.add_option('-f', '--filename', help='Filename',dest='filename')
	parser.add_option('-d', '--dataset', help='Dataset',dest='userDataset')
	(options,args) = parser.parse_args()
	if not options.filename and not options.userDataset:
		print "A filename or dataset is required"
		sys.exit(0)

        filename=options.filename

        if options.filename:
           f=open(filename,'r')
        else:
           f=[options.userDataset]

        for dataset in f:
           dataset = dataset.rstrip('\n')
           status = dbs3.getDatasetStatus(dataset)
           print dataset, status

	sys.exit(0)
예제 #8
0
def main():
    url = 'cmsweb.cern.ch'
    parser = optparse.OptionParser()
    parser.add_option('-f', '--filename', help='Filename', dest='filename')
    parser.add_option('-d', '--dataset', help='Dataset', dest='userDataset')
    (options, args) = parser.parse_args()
    if not options.filename and not options.userDataset:
        print "A filename or dataset is required"
        sys.exit(0)

    filename = options.filename

    if options.filename:
        f = open(filename, 'r')
    else:
        f = [options.userDataset]

    for dataset in f:
        dataset = dataset.rstrip('\n')
        status = dbs3.getDatasetStatus(dataset)
        print dataset, status

    sys.exit(0)