Пример #1
0
def requestDeletions(site,dsetNames):
    phedex = phedexApi.phedexApi(logPath='./')
    # compose data for deletion request
    check,data = phedex.xmlData(datasets=dsetNames,instance='prod',level='block')
    if check:
        print " ERROR - phedexApi.xmlData failed"
        sys.exit(1)

    # here the request is really sent
    message = 'IntelROCCS -- Cache Release Request -- Deprecated Datasets'
    check,response = phedex.delete(node=site,data=data,comments=message,instance='prod')
    if check:
        print " ERROR - phedexApi.delete failed"
        print site
        print response
        sys.exit(1)
    respo = response.read()
    matchObj = re.search(r'"id":"(\d+)"',respo)
    reqid = int(matchObj.group(1))
    del phedex

     # here we brute force deletion to be approved
    phedex = phedexApi.phedexApi(logPath='./')
    check,response = phedex.updateRequest(decision='approve',request=reqid,node=site,instance='prod')
    if check:
        print " ERROR - phedexApi.updateRequest failed - reqid="+ str(reqid)
        print response
    del phedex
Пример #2
0
    def submitTransferRequest(self,site,datasets2trans):
        if len(datasets2trans) < 1:
            return
        phedex = phedexApi.phedexApi(logPath='./')
        # compose data for deletion request
        check,data = phedex.xmlData(datasets=datasets2trans,instance='prod')
        if check:
            print " ERROR - phedexApi.xmlData failed"
            print data
            sys.exit(1)
            
        # here the request is really sent
        message = 'IntelROCCS -- Automatic Transfer Request'
        check,response = phedex.subscribe(node=site,data=data,comments=message)
        if check:
            print " ERROR - phedexApi.subscribe failed"
            print response
            sys.exit(1)

        respo = response.read()
        matchObj = re.search(r'"id":"(\d+)"',respo)
        reqid = int(matchObj.group(1))

        rdate = (re.search(r'"request_date":"(.*?)"',respo)).group(1)
        rdate = rdate[:-3]
        self.dbInfoHandler.logRequest(site,datasets2trans,reqid,rdate,0)
Пример #3
0
    def submitDeletionRequest(self,site,datasets2del):
        if len(datasets2del) < 1:
            return
        phedex = phedexApi.phedexApi(logPath='./')
        # compose data for deletion request
        check,data = phedex.xmlData(datasets=datasets2del,instance='prod')
        if check:
            print " ERROR - phedexApi.xmlData failed"
            sys.exit(1)
            
        # here the request is really sent
        message = 'IntelROCCS -- Automatic Cache Release Request (next check ' + \
            'in about %s hours).'%(os.environ['DETOX_CYCLE_HOURS']) + \
            ' Summary at: http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/result/'
        check,response = phedex.delete(node=site,data=data,comments=message,instance='prod')
        if check:
            print " ERROR - phedexApi.delete failed"
            print response
            sys.exit(1)

        respo = response.read()
        matchObj = re.search(r'"id":"(\d+)"',respo)
        reqid = int(matchObj.group(1))

        rdate = (re.search(r'"request_date":"(.*?)"',respo)).group(1)
        rdate = rdate[:-3]
        self.dbInforhandler.logRequest(site,datasets2del,reqid,rdate,1)
Пример #4
0
    def submitTransferRequest(self, site, datasets2trans):
        if len(datasets2trans) < 1:
            return
        phedex = phedexApi.phedexApi(logPath='./')
        # compose data for deletion request
        check, data = phedex.xmlData(datasets=datasets2trans, instance='prod')
        if check:
            print " ERROR - phedexApi.xmlData failed"
            print data
            sys.exit(1)

        # here the request is really sent
        message = 'IntelROCCS -- Automatic Transfer Request'
        check, response = phedex.subscribe(node=site,
                                           data=data,
                                           comments=message)
        if check:
            print " ERROR - phedexApi.subscribe failed"
            print response
            sys.exit(1)

        respo = response.read()
        matchObj = re.search(r'"id":"(\d+)"', respo)
        reqid = int(matchObj.group(1))

        rdate = (re.search(r'"request_date":"(.*?)"', respo)).group(1)
        rdate = rdate[:-3]
        self.dbInfoHandler.logRequest(site, datasets2trans, reqid, rdate, 0)
Пример #5
0
    def submitDeletionRequest(self, site, datasets2del):
        if len(datasets2del) < 1:
            return
        phedex = phedexApi.phedexApi(logPath='./')
        # compose data for deletion request
        check, data = phedex.xmlData(datasets=datasets2del, instance='prod')
        if check:
            print " ERROR - phedexApi.xmlData failed"
            sys.exit(1)

        # here the request is really sent
        message = 'IntelROCCS -- Automatic Cache Release Request (next check ' + \
            'in about %s hours).'%(os.environ['DETOX_CYCLE_HOURS']) + \
            ' Summary at: http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/result/'
        check, response = phedex.delete(node=site,
                                        data=data,
                                        comments=message,
                                        instance='prod')
        if check:
            print " ERROR - phedexApi.delete failed"
            print response
            sys.exit(1)

        respo = response.read()
        matchObj = re.search(r'"id":"(\d+)"', respo)
        reqid = int(matchObj.group(1))

        rdate = (re.search(r'"request_date":"(.*?)"', respo)).group(1)
        rdate = rdate[:-3]
        self.dbInforhandler.logRequest(site, datasets2del, reqid, rdate, 1)
Пример #6
0
 def submitUpdateRequest(self,site,reqid):
     # here we brute force deletion to be approved
     phedex = phedexApi.phedexApi(logPath='./')
     check,response = phedex.updateRequest(decision='approve',request=reqid,node=site,instance='prod')
     if check:
         print " ERROR - phedexApi.updateRequest failed - reqid="+ str(reqid)
         print response
     del phedex
Пример #7
0
 def submitUpdateRequest(self,site,reqid):
     # here we brute force deletion to be approved
     phedex = phedexApi.phedexApi(logPath='./')
     check,response = phedex.updateRequest(decision='approve',request=reqid,node=site,instance='prod')
     if check:
         print " ERROR - phedexApi.updateRequest failed - reqid="+ str(reqid)
         print response
     del phedex
Пример #8
0
def submitRequest(site, datasets=[]):

    if len(datasets) < 1:
        print " ERROR - Trying to submit empty request for " + site
        return

    phedex = phedexApi.phedexApi(logPath='./')

    # compose data for deletion request
    check,data = phedex.xmlData(datasets=datasets,instance='prod')

    if check: 
        print " ERROR - phedexApi.xmlData failed"
        sys.exit(1)
    
    # here the request is really sent
    message = 'IntelROCCS -- Automatic Cache Release Request (if not acted upon will repeat ' + \
              'in about %s hours).'%(os.environ['DETOX_CYCLE_HOURS']) + \
              ' Summary at: http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/result/'
    check,response = phedex.delete(node=site,data=data,comments=message,instance='prod')
    if check:
        print " ERROR - phedexApi.delete failed"
        print response
        sys.exit(1)

    respo = response.read()
    matchObj = re.search(r'"id":"(\d+)"',respo)
    id = int(matchObj.group(1))
    
    matchObj = re.search(r'"request_date":"(.*?)"',respo)
    date = matchObj.group(1)
    date = date[:-3]
    myCnf = os.environ['DETOX_MYSQL_CONFIG']

    for dataset in datasets:
        rank = float(datasetInfo[dataset][0])
        size = float(datasetInfo[dataset][1])
        group = datasetGroup[dataset]
        
        db = MySQLdb.connect(read_default_file=myCnf,read_default_group="mysql")
        cursor = db.cursor()
        sql = "insert into Requests(RequestId,RequestType,SiteName,Dataset,Size,Rank,GroupName," + \
              "TimeStamp) values ('%d', '%d', '%s', '%s', '%d', '%d', '%s', '%s' )" % \
              (id, 1, site, dataset,size,rank,group,date)

        # ! this could be done in one line but it is just nice to see what is deleted !
        try:
            cursor.execute(sql)
            db.commit()
        except:
            print "caught an exception"
            db.rollback()
        
        db.close()
Пример #9
0
def requestDeletions(site, dsetNames):
    phedex = phedexApi.phedexApi(logPath='./')
    # compose data for deletion request
    check, data = phedex.xmlData(datasets=dsetNames,
                                 instance='prod',
                                 level='block')
    if check:
        print " ERROR - phedexApi.xmlData failed"
        sys.exit(1)

    # here the request is really sent
    message = 'IntelROCCS -- Cache Release Request -- Deprecated Datasets'
    check, response = phedex.delete(node=site,
                                    data=data,
                                    comments=message,
                                    instance='prod')
    if check:
        print " ERROR - phedexApi.delete failed"
        print site
        print response
        sys.exit(1)
    respo = response.read()
    matchObj = re.search(r'"id":"(\d+)"', respo)
    reqid = int(matchObj.group(1))
    del phedex

    # here we brute force deletion to be approved
    phedex = phedexApi.phedexApi(logPath='./')
    check, response = phedex.updateRequest(decision='approve',
                                           request=reqid,
                                           node=site,
                                           instance='prod')
    if check:
        print " ERROR - phedexApi.updateRequest failed - reqid=" + str(reqid)
        print response
    del phedex
Пример #10
0
    def printRunawaySets(self):
        siteSizes = {}
        siteSets = {}
        for dset in self.phedexDatasets:
            if dset not in self.otherDatasets:
                continue

            dataset = self.otherDatasets[dset]
            for site in dataset.siteNames:
                group = dataset.group(site)
                size = dataset.size(site)
                if site not in siteSizes:
                    siteSizes[site] = 0
                    siteSets[site] = 0
                siteSizes[site] = siteSizes[site] + size / 1000
                siteSets[site] = siteSets[site] + 1
                if site not in self.runAwayGroups:
                    dsetName = dataset.dataset
                    if 'AOD' in dsetName:
                        #                        if group != 'RelVal' and group != 'DataOps':
                        if not site.startswith('T1_'):
                            if group == 'local':
                                continue
                            print site
                            print dataset.dataset
                            print group + '--> AnalysisOps'
                            phedex = phedexApi.phedexApi(logPath='./')
                            check, response = phedex.changeGroup(
                                site, dsetName, 'AnalysisOps')
                            if check:
                                print " ERROR - phedexApi.updateRequest failed"
                                print response
                                del phedex
                            else:
                                del phedex
                                continue

                    self.runAwayGroups[site] = [group]
                else:
                    if group not in self.runAwayGroups[site]:
                        self.runAwayGroups[site].append(group)

        if (len(siteSizes) < 1):
            return
        print " !! WARNING !! - those sites have datasets in wrong groups"
        for site in sorted(siteSizes):
            print ' %3d %6.2f TB' % (siteSets[site],
                                     siteSizes[site]) + ": " + site
Пример #11
0
    def printRunawaySets(self):
        siteSizes = {}
        siteSets = {}
        for dset in self.phedexDatasets:
            if dset not in self.otherDatasets:
                continue

            dataset = self.otherDatasets[dset]
            for site in dataset.siteNames:
                group = dataset.group(site)
                size  = dataset.size(site)
                if site not in siteSizes:
                    siteSizes[site] = 0
                    siteSets[site] = 0
                siteSizes[site] = siteSizes[site] + size/1000
                siteSets[site] = siteSets[site] + 1
                if site not in self.runAwayGroups:
                    dsetName = dataset.dataset
                    if 'AOD' in dsetName:
#                        if group != 'RelVal' and group != 'DataOps':
                         if not site.startswith('T1_'):
                            if group == 'local':
                               continue
                            print site
                            print dataset.dataset
                            print group + '--> AnalysisOps'
                            phedex = phedexApi.phedexApi(logPath='./')
                            check,response = phedex.changeGroup(site,dsetName,'AnalysisOps')
                            if check:
                                print " ERROR - phedexApi.updateRequest failed"
                                print response
                                del phedex
                            else:
                                del phedex
                                continue

                    self.runAwayGroups[site] = [group]
                else:
                    if group not in self.runAwayGroups[site]:
                        self.runAwayGroups[site].append(group)

        if (len(siteSizes) < 1):
            return
        print " !! WARNING !! - those sites have datasets in wrong groups"
        for site in sorted(siteSizes):
            print ' %3d %6.2f TB'%(siteSets[site],siteSizes[site]) + ": " + site
Пример #12
0
 def __init__(self):
     config = ConfigParser.RawConfigParser()
     config.read(os.path.join(os.path.dirname(__file__), 'api.cfg'))
     self.phedexCache = config.get('phedex', 'cache')
     self.cacheDeadline = config.getint('phedex', 'expiration_timer')
     self.phedexApi = phedexApi.phedexApi()
Пример #13
0
 def __init__(self):
     config = ConfigParser.RawConfigParser()
     config.read(os.path.join(os.path.dirname(__file__), 'api.cfg'))
     self.phedexCache = config.get('phedex', 'cache')
     self.cacheDeadline = config.getint('phedex', 'expiration_timer')
     self.phedexApi = phedexApi.phedexApi()