def __init__(self): user = config.get_config('db_user', type='str') password = config.get_config('db_password', type='str') connectionName = config.get_config('db_conn', type='str') if not user or not password or not connectionName: raise Exception('No DB connection specified in configuration file') connection_string = '%s/%s@%s'%(user, password, connectionName) self.__connection = cx_Oracle.Connection(connection_string) self.__logger = logging.getLogger('dq2.victor.victorDao')
def __init__(self): user = config.get_config('db_user', type='str') password = config.get_config('db_password', type='str') connectionName = config.get_config('db_conn', type='str') if not user or not password or not connectionName: raise Exception('No DB connection specified in configuration file') connection_string = '%s/%s@%s' % (user, password, connectionName) self.__connection = cx_Oracle.Connection(connection_string) self.__logger = logging.getLogger('dq2.victor.victorDao')
class Victor: """ Class for cleaning agent It will monitor sites which have content over total space ratio above some given value and clean dataset replicas from these overflowed sites according to algorithm provided in DatasetusageMonitir class """ __logger = logging.getLogger("dq2.victor") __period = HOUR __victorDao = VictorDao() def __init__(self, name, configFile): """ Initializer for the object. """ self.__logger.info('Victor starting up...') self.__siteSelection = SiteSelection() self.__replicaReduction = ReplicaReduction() def run(self): try: self.__victorDao.insertRun() #Development: Uncomment later fullSiteInfo, accountingSummary = self.__siteSelection.getFullSites() #fullSiteInfo = {'SARA-MATRIX_DATADISK': (0.01, 990*(10**12), 1000*(10**12), 0, 0, True, 100**12, 'DATADISK')} #fullSiteInfo = {u'T2_CH_CSCS^b-physics': (-0.19219963159204001, 119219963159204, 100000000000000.0, 0, 0, True, 49219963159204.008, None)} #accountingSummary = { u'T2_CH_CSCS^b-physics': {'total': 100000000000000.0, 'tobedeleted': 0, 'used': 119219963159204, 'indeletionqueue': 0, 'newlycleaned': 0}} #fullSiteInfo = {'T2_US_Vanderbilt^heavy-ions': fullSiteInfo['T2_US_Vanderbilt^heavy-ions'], 'T2_BE_UCL^AnalysisOps': fullSiteInfo['T2_BE_UCL^AnalysisOps'],} #accountingSummary = {'T2_US_Vanderbilt^heavy-ions': accountingSummary['T2_US_Vanderbilt^heavy-ions'], 'T2_BE_UCL^AnalysisOps': accountingSummary['T2_BE_UCL^AnalysisOps']} datasetsForSite, successfullycleaned, accountingSummary = self.__replicaReduction.cleanSites(fullSiteInfo, accountingSummary) prepareSummary(accountingSummary) self.__victorDao.insertAccountingSummary(accountingSummary) self.__victorDao.closeRun() except Exception as e: print(traceback.format_exc()) self.__logger.critical(traceback.format_exc())
def __init__ (self, query=False, remote=False): """ Class constructor. @author: Miguel Branco @contact: [email protected] @since: 0.3 @version: $Id: DQDashboardTool.py,v 1.1 2008-05-29 11:15:39 psalgado Exp $ @param query: Boolean indicating if default query options are available or not @param remote: Boolean indicating if default remote access (service, port) options are available or not """ self.__class__._logger = logging.getLogger(self.__module__) self.parser = OptionParser(usage=self.usage, version=self.version, description=self.description) # default options are always available options = self.defaultOptions # if this is a 'query' tool, default query options are available if query: options.extend(self.defaultQueryOptions) # if this is a 'remote access' tool, service, port and alike options # are also available if remote: options.extend(self.remoteOptions) # finally, tool specific options should be added options.extend(self.toolOptions) # add all the options to the parser for option in options: self.parser.add_option(option) # parse the command line arguments and options and update the options (self.options, self.args) = self.parser.parse_args()
class AccountingInterface(AccountingInterface): """ Class encapsulating the CMS Storage Information Interface. """ __freeRatio = {'default': 0.1} __freeAbsolute = {'default': 25 * TERA} __targetRatio = {'default': 0.2} __targetAbsolute = {} __specifiedSpacetokens = [] __centralPledge = 0.7 threshold_per_site = {} __logger = logging.getLogger("dq2.victor.AccountingInterface") #__configurationInterface='DQ2ConfigurationInterface' def __init__(self, name): self.__nodeusage = self.__get_nodeusage_values() self.__groupusage = self.__get_groupusage_values() self.__logger.debug('NODE USAGE: %s' % self.__nodeusage) self.__logger.debug('GROUP USAGE: %s' % self.__groupusage) freeRatio = config.get_dict('freeRatio', type=float) if freeRatio is not None: self.__freeRatio = freeRatio freeAbsolute = config.get_dict('freeAbsolute', type='size') if freeAbsolute is not None: self.__freeAbsolute = freeAbsolute targetRatio = config.get_dict('targetRatio', type=float) if targetRatio is not None: self.__targetRatio = targetRatio targetAbsolute = config.get_dict('targetAbsolute', type='size') if targetAbsolute is not None: self.__targetAbsolute = targetAbsolute centralPledge = config.get_dict('centralPledge', type=float) if centralPledge is not None: self.__centralPledge = centralPledge self.__site_name_map = self.__get_site_name_map() self.__sitedb_pledges = self.__get_sitedb_pledges() def __get_site_name_map(self): url = 'https://cmsweb.cern.ch/sitedb/data/prod/site-names' site_map_data = get_json_data_https(url) siteIdx = site_map_data['desc']['columns'].index('site_name') typeIdx = site_map_data['desc']['columns'].index('type') nameIdx = site_map_data['desc']['columns'].index('alias') site_name_map = {} for item in site_map_data['result']: try: (site_name_map[item[siteIdx]][item[typeIdx]]).append( item[nameIdx]) except KeyError: try: site_name_map[item[siteIdx]][item[typeIdx]] = [ item[nameIdx] ] except KeyError: site_name_map[item[siteIdx]] = { item[typeIdx]: [item[nameIdx]] } self.__logger.debug(site_name_map) return site_name_map def __get_sitedb_pledges(self): url = 'https://cmsweb.cern.ch/sitedb/data/prod/resource-pledges' resource_pledge_data = get_json_data_https(url) siteIdx = resource_pledge_data['desc']['columns'].index('site_name') updateTimeIdx = resource_pledge_data['desc']['columns'].index( 'pledge_date') yearIdx = resource_pledge_data['desc']['columns'].index('quarter') diskIdx = resource_pledge_data['desc']['columns'].index('disk_store') localDiskIdx = resource_pledge_data['desc']['columns'].index( 'local_store') resource_pledges = {} current_year = time.gmtime().tm_year for item in resource_pledge_data['result']: if item[yearIdx] == current_year: try: if item[updateTimeIdx] >= resource_pledges[ item[siteIdx]]['pledge_date']: resource_pledges[ item[siteIdx]]['disk_store'] = item[diskIdx] resource_pledges[ item[siteIdx]]['local_store'] = item[localDiskIdx] resource_pledges[ item[siteIdx]]['pledge_date'] = item[updateTimeIdx] except KeyError: resource_pledges[item[siteIdx]] = { 'disk_store': item[diskIdx], 'local_store': item[localDiskIdx], 'pledge_date': item[updateTimeIdx] } self.__logger.debug(resource_pledges) return resource_pledges def __get_nodeusage_values(self): url = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/nodeusage" nodeusage_aux = get_json_data(url) nodeusage = {} #Massage information to make it more accessible for site_dic in nodeusage_aux['phedex']['node']: site = site_dic["name"] try: nodeusage[site] = { "src_node_bytes": int(site_dic["src_node_bytes"]), "nonsrc_node_bytes": int(site_dic["nonsrc_node_bytes"]), "noncust_dest_bytes": int(site_dic["noncust_dest_bytes"]), "noncust_node_bytes": int(site_dic["noncust_node_bytes"]), "cust_node_bytes": int(site_dic["cust_node_bytes"]), "cust_dest_bytes": int(site_dic["cust_dest_bytes"]) } except: self.__logger.error( 'Incomplete information for site %s in %s' % (site, url)) return nodeusage def __get_groupusage_values(self): url = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/groupusage" groupusage_aux = get_json_data(url) groupusage = {} #Massage information to make it more accessible for site_dic in groupusage_aux['phedex']['node']: site = site_dic["name"] groupusage[site] = {} for group_dic in site_dic['group']: group = group_dic["name"] try: groupusage[site][group] = { "dest_bytes": int(group_dic["dest_bytes"]), "node_bytes": int(group_dic["node_bytes"]) } except: self.__logger.error( 'Incomplete information for group %s in site %s in %s' % (group, site, url)) return groupusage def calculateSpaceToClean(self, site, used, total, tobedeleted, indeletionqueue): site, physicsgroup = site.split('^') targetAbsolute = 25 * TERA targetRatio = 0.3 bigsiteThreshold = 100 * TERA if total > bigsiteThreshold: targetValue = total - targetAbsolute #Targetvalue is of used diskspace else: targetValue = total * (1 - targetRatio ) #Targetvalue is of used diskspace effectivelyused = used - tobedeleted - indeletionqueue tobecleaned = effectivelyused - targetValue return tobecleaned, None def getSites(self, spacetokens=[]): """ @return: List with all the sites. """ sites = [] for site in self.__site_name_map: site_name = self.__site_name_map[site]['cms'][0] try: groups = groupPledges[site_name].keys() for group in groups: sites.append('%s^%s' % (site_name, group)) except KeyError: pass return sites def __getUsedSpaceCentral(self, site): try: #Basic usage of storage bytes = self.__nodeusage[site][ 'noncust_node_bytes'] + self.__nodeusage[site][ 'src_node_bytes'] + self.__nodeusage[site][ 'nonsrc_node_bytes'] #Add reserved space of ongoing transfers #bytes = bytes + self.__nodeusage[site]['noncust_dest_bytes'] #THE PREVIOUS CASE IS ONLY VALID FOR T2s!!! self.__logger.info('Used space for site %s: %.2f' % (site, bytes / TERA)) return bytes except Exception as e: self.__logger.error('No used space for site %s (%s)' % (site, e)) return None def __getUsedSpaceGroup(self, site, group): try: #Basic usage of storage bytes = self.__groupusage[site][group]['node_bytes'] self.__logger.info('Used space for site %s by group %s: %.2f' % (site, group, bytes / TERA)) return bytes except Exception as e: self.__logger.error('No used space for site %s by group %s (%s)' % (site, group, e)) return None def getUsedSpace(self, site): site, physicsgroup = site.split('^') return self.__getUsedSpaceGroup(site, physicsgroup) def __getSiteDBPledge(self, site): for item in self.__site_name_map: if self.__site_name_map[item]['cms'][0] == site: siteId = item try: pledge = ((self.__sitedb_pledges[siteId]['disk_store'] or 0) - (self.__sitedb_pledges[siteId]['local_store'] or 0)) * TERA * self.__centralPledge if pledge is not None: self.__logger.info('Pledged space for site %s: %.2f' % (site, pledge / TERA)) return pledge except KeyError: self.__logger.error('No pledge for site %s' % (site)) return None def __getGroupPledge(self, site, group): try: bytes = groupPledges[site][group] self.__logger.info('Pledged space for site %s by group %s: %.2f' % (site, group, bytes / TERA)) return bytes except Exception as e: self.__logger.error( 'No pledged space for site %s by group %s (%s)' % (site, group, e)) return None def getTotalSpace(self, site): site, physicsgroup = site.split('^') return self.__getGroupPledge(site, physicsgroup) def getToBeDeletedSpace(self, site): """ MOCK. Not needed by CMS """ return 0 def refreshSiteStatistics(self, site): """ MOCK. Not needed by CMS """ return True def getSpaceInDeletionQueue(self, site): ''' Return space which is freed up by deletion ''' return 0 #try: #Basic usage of storage # bytes = self.__nodeusage[site]['src_node_bytes'] + self.__nodeusage[site]['nonsrc_node_bytes'] # self.__logger.info('Space in deletion queue for site %s: %.2f' %(site, bytes/TERA)) # return bytes #except: # self.__logger.error('No space in deletion queue for site %s' %(site)) # return 0 #It has to be 0, not None! def isFull(self, site, ratio, free): thresholdFree = 0.1 thresholdAbsFree = 15 * TERA if ratio > thresholdFree or free > thresholdAbsFree: return False return True
class PopularityInterface(PopularityInterface): """ Class encapsulating the CMS Popularity Interface. """ __logger = logging.getLogger("dq2.victor.PopularityInterface") __MONTH = 30 __REPLICA_CREATION_DATE_COL = 4 def __init__(self, name): #Create a temporary directory to cache information self.__tmpdirectory = tempfile.mkdtemp() def __hasCustodialCopy(self, blockname): url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/blockreplicas?custodial=y&complete=y&block=%s' % ( blockname) url = url.replace('#', '%23') self.__logger.debug(url) replicas = get_json_data(url) ''' replicas = { "phedex":{ "request_version":"2.1.8", "request_timestamp":1311755259.31146, "instance":"prod", "request_call":"blockreplicas", "request_url":"http://cmsweb.cern.ch:7001/phedex/datasvc/json/prod/blockreplicas", "request_date":"2011-07-27 08:27:39 UTC", "block":[ { "bytes":"67095800951", "files":"32", "is_open":"n", "name":"/ZJetsToNuNu_Pt-100_7TeV-herwigpp/Summer11-START311_V2-v1/GEN-SIM#7f6b861b-2263-4854-8abf-d096d35d9f1a", "id":"2576551", "replica":[ { "bytes":"67095800951", "files":"32", "node":"T1_IT_CNAF_MSS", "time_create":"1311331011", "time_update":"1311610457.398", "group":"DataOps", "node_id":"8", "custodial":"y", "se":"storm-fe-cms.cr.cnaf.infn.it", "subscribed":"y", "complete":"y"}]}], "call_time":"0.05906"}} ''' try: if replicas['phedex']['block'][0]['replica']: return True except KeyError: self.__logger.warning( 'Block %s excepted with KeyError. replicas = %s' % (blockname, replicas)) return False except IndexError: self.__logger.warning( 'Block %s excepted with IndexError. replicas = %s' % (blockname, replicas)) return False return False def __getDatasetStats(self, blocks): datasetStats = {} for blockname in blocks: datasetName = blockname.split('#')[0] datasetStats.setdefault(datasetName, { 'max': -1, 'sum': 0, 'nblocks': 0 }) datasetStats[datasetName]['nblocks'] += 1 datasetStats[datasetName]['sum'] += int( blocks[blockname]['popularitynacc']) if datasetStats[datasetName]['max'] < int( blocks[blockname]['popularitynacc']): datasetStats[datasetName]['max'] = int( blocks[blockname]['popularitynacc']) return datasetStats def __validateBlock(self, blockname, blockinfo, group, maxnacc, maxcdate): if blockinfo['custodial'] != 'n': #self.__logger.debug('Refuse because CUSTODIAL') return False if blockinfo['group'] != group: #self.__logger.debug('Refuse because we look for replicas from %s and the replica belongs to %s' %(group, blockinfo['group'])) return False if blockinfo['popularitynacc'] > maxnacc: self.__logger.debug( 'Refuse because replica has been accessed %s times (>%s)' % (blockinfo['popularitynacc'], maxnacc)) return False cdate = datetime.datetime.fromtimestamp(blockinfo['creation_time']) cdate_max = datetime.datetime.now() - datetime.timedelta(days=maxcdate) if cdate > cdate_max: self.__logger.debug( 'Refuse because replica is too young %s (>%s)' % (cdate, cdate_max)) return False if not self.__hasCustodialCopy(blockname): self.__logger.debug('Refuse because replica has no custodial copy') return False return True def __parseUnpopularBlocks(self, site, unpopularBlocks, threshold, creationlimit, physicsgroup): startdate = unpopularBlocks['popularitytstart'] enddate = unpopularBlocks['popularitytstop'] blocks = unpopularBlocks[site] blocks_list = [] datasetStats = self.__getDatasetStats(blocks) for blockname in blocks: datasetname = blockname.split('#')[0] #Skip custodial blocks, blocks that don't belong to central AnalysisOps and blocks without a custodial copy #if blocks[blockname]['custodial']=='n' and blocks[blockname]['group']=='AnalysisOps' and self.__hasCustodialCopy(blockname): if self.__validateBlock(blockname, blocks[blockname], physicsgroup, threshold, creationlimit): try: new_block = (str(blockname), int(blocks[blockname]['popularitynacc']), None, datetime.datetime.fromtimestamp( blocks[blockname]['creation_time']), datetime.datetime.fromtimestamp( blocks[blockname]['creation_time']), int(blocks[blockname]['size']), int(blocks[blockname]['popularitycpu']), int(blocks[blockname]['popularitynacc']), datasetStats[datasetname]['nblocks'], datasetStats[datasetname]['max'], datasetStats[datasetname]['sum']) blocks_list.append(new_block) self.__logger.debug('Appended %s to %s: %s' % (blockname, site, new_block)) except: self.__logger.error( 'Error processing block %s for site %s. Complete information: %s' % (blockname, site, blocks[blockname])) continue else: self.__logger.debug('Refused %s for %s' % (blockname, site)) #self.__logger.debug('Block %s skipped: custodial %s and group %s' %(blockname, blocks[blockname]['custodial'], blocks[blockname]['group'])) return blocks_list def __getBlocks(self, site): if os.path.isfile('%s/blocks_%s' % (self.__tmpdirectory, site)): unpopularBlocks = get_json_data_from_file( '%s/blocks_%s' % (self.__tmpdirectory, site)) return unpopularBlocks else: url = 'https://cmsweb.cern.ch/popdb/victorinterface/popdbcombine/?sitename=%s' % ( site) unpopularBlocks = get_json_data_https(url) dumpTemporaryInfo(unpopularBlocks, self.__tmpdirectory, 'blocks_%s' % (site)) return unpopularBlocks def getUnpopularDatasets(self, site, lastaccess, threshold, creationlimit, replicatype='secondary'): ''' Get the list of a site's unpopular block replicas according to the input criteria. ''' def compdate(x, y): COMPARE_COL = self.__REPLICA_CREATION_DATE_COL if epochTime(x[COMPARE_COL]) > epochTime(y[COMPARE_COL]): return 1 elif epochTime(x[COMPARE_COL]) < epochTime(y[COMPARE_COL]): return -1 else: return 0 site, physicsgroup = site.split('^') st = time.time() self.__logger.info('Getting unpopular datasets for %s...' % site) try: blocks = self.__getBlocks(site) if not blocks or not blocks[site]: raise Exception("Empty answer received from unpopularity API") unpopularBlocksListed = self.__parseUnpopularBlocks( site, blocks, threshold, creationlimit, physicsgroup) except Exception as e: self.__logger.critical('Failed to process site %s [%s - %s]' % (site, e, traceback.format_exc())) sendErrorMail('%s\n%s' % (e, traceback.format_exc())) return {} finally: elapsed = round(time.time() - st, 2) self.__logger.info( '...back from popDB API for site %s [Took %.2f seconds]' % (site, elapsed)) unpopularBlocksListed = sorted(unpopularBlocksListed, compdate) return unpopularBlocksListed
class SiteSelection: __logger = logging.getLogger("dq2.victor.siteSelection") __accountingInterface = 'DQ2AccountingInterface' def __init__(self): accountingInterface = config.get_config('accountingInterface', type=str) if accountingInterface: self.__accountingInterface = accountingInterface self.__accounting = create_tool(self.__accountingInterface) def getFullSites(self): sitestoinspect = self.__accounting.getSites() fullSiteInfo = {} allInfo = {} self.__logger.info('Sites to check for full storages: %s' % sitestoinspect) #sitestoinspect = sitestoinspect[0:5] for site in sitestoinspect: self.__logger.debug('Going to evaluate site %s' % (site)) refreshed = self.__accounting.refreshSiteStatistics(site) used = self.__accounting.getUsedSpace(site) total = self.__accounting.getTotalSpace(site) tobedeleted = self.__accounting.getToBeDeletedSpace(site) indeletionqueue = self.__accounting.getSpaceInDeletionQueue(site) allInfo[site] = { 'used': used, 'total': total, 'tobedeleted': tobedeleted, 'indeletionqueue': indeletionqueue, 'newlycleaned': 0 } if used == None or total == None or total == 0: self.__logger.warning( 'Skipped site: %s - Missing values for used and/or pledged space' % (site)) continue free = total - used + tobedeleted + indeletionqueue ratio = free * 1.0 / total self.__logger.info( 'Accounting Summary. Site: %s - Used: %.2fTB Free: %.2fTB (%.2f) Total: %.2fTB ToBeDeleted: %.2fTB InDeletionQueue %.2fTB RefreshedStatistics: %s' % (site, used / TERA, free / TERA, ratio, total / TERA, tobedeleted / TERA, indeletionqueue / TERA, refreshed)) if not self.__accounting.isFull(site, ratio, free): self.__logger.debug('Site %s is not full' % (site)) continue spacetoclean, spacetoken = self.__accounting.calculateSpaceToClean( site, used, total, tobedeleted, indeletionqueue) fullSiteInfo[site] = (ratio, used, total, tobedeleted, indeletionqueue, refreshed, spacetoclean, spacetoken) thresholdInfo = self.__accounting.threshold_per_site return fullSiteInfo, allInfo
""" @copyright: European Organization for Nuclear Research (CERN) @author: Andrii Thykonov U{[email protected]<mailto:[email protected]>}, CERN, 2010-2011 @author: Fernando H. Barreiro U{[email protected]<mailto:[email protected]>}, CERN, 2011 @license: Licensed under the Apache License, Version 2.0 (the "License"); You may not use this file except in compliance with the License. You may obtain a copy of the License at U{http://www.apache.org/licenses/LICENSE-2.0} """ from dq2.common import log as logging from dq2.victor import config import urllib2, httplib import simplejson logger = logging.getLogger("dq2.victor.utils") class HTTPSClientAuthHandler(urllib2.HTTPSHandler): """ Simple HTTPS client authentication class based on provided key/ca information """ def __init__(self, key=None, cert=None, level=0): if level > 1: urllib2.HTTPSHandler.__init__(self, debuglevel=1) else: urllib2.HTTPSHandler.__init__(self) self.key = key self.cert = cert
class Timer: """ Class with Borg design pattern: http://code.activestate.com/recipes/66531-singleton-we-dont-need-no-stinkin-singleton-the-bo/ The dictionary """ __timings = { 'accounting': { 'calls': 0, 'time': 0 }, 'deletedVol': { 'calls': 0, 'time': 0 }, 'refreshAcc': { 'calls': 0, 'time': 0 }, 'popularity': { 'calls': 0, 'time': 0 }, 'setMetadata': { 'calls': 0, 'time': 0 }, 'thresholds': { 'calls': 0, 'time': 0 } } __logger = logging.getLogger("dq2.victor.timer") def __init__(self): self.__dict__ = self.__shared_state def increaseAccounting(self, time): self.__timings['accounting']['calls'] += 1 self.__timings['accounting']['time'] += time def increaseDeletedVolume(self, time): self.__timings['deletedVol']['calls'] += 1 self.__timings['deletedVol']['time'] += time def increasePopularity(self, time): self.__timings['popularity']['calls'] += 1 self.__timings['popularity']['time'] += time def increaseSetMetadata(self, time): self.__timings['setMetadata']['calls'] += 1 self.__timings['setMetadata']['time'] += time def increaseThresholds(self, time): self.__timings['thresholds']['calls'] += 1 self.__timings['thresholds']['time'] += time def printout(self): self.__logger.info(""" Accounting calls: %d in %.1f DeletedVol calls: %d in %.1f RefreshAcc calls: %d in %.1f Popularity calls: %d in %.1f setMetadata calls: %d in %.1f Thresholds calls: %d in %.1f """ % (self.__timings['accounting']['calls'], self.__timings['accounting']['time'], self.__timings['deletedVol']['calls'], self.__timings['deletedVol']['time'], self.__timings['refreshAcc']['calls'], self.__timings['refreshAcc']['time'], self.__timings['popularity']['calls'], self.__timings['popularity']['time'], self.__timings['setMetadata']['calls'], self.__timings['setMetadata']['time'], self.__timings['thresholds']['calls'], self.__timings['thresholds']['time']))
class DatasetSelection: __logger = logging.getLogger("dq2.victor.datasetSelection") __popularityInterface = 'DQ2PopularityInterface' MONTH = 30 # Algorithm 1. Default cleaning algorithm #__thresholds = [{'threshold' : 1, 'lastaccess' : [3, 2, 1]}] __thresholds = [{'threshold': 1, 'lastaccess': [1]}] __creationlimit = 90 #days # Algorithm 2. User spacetokens and similar. Delete by replica creation date __algorithm2_custodiality = 'default' __algorithm2_spacetokens = [] #'SCRATCHDISK'] __algorithm2_creationlimit = {} #{'SCRATCHDISK':15, 'PRODDISK':31} #Positions of __DATASET_NAME_COL = 0 __LASTACCESS_COL = 2 __DATASET_CREATION_DATE_COL = 3 __REPLICA_CREATION_DATE_COL = 4 __DATASET_SIZE_COL = 5 __CPU_COL = 6 __NACC_COL = 7 __NBLOCKS_COL = 8 __NMAX_COL = 9 __NTOTAL_COL = 10 def __init__(self): popularityInterface = config.get_config('popularityInterface', type=str) if popularityInterface: self.__popularityInterface = popularityInterface self.__popularity = create_tool(self.__popularityInterface) creationlimit = config.get_config('creationlimit', int) if creationlimit is not None: self.__creationlimit = creationlimit thresholds = config.get_config('thresholds', list) lastaccess = config.get_config('lastaccess', 'list_2') is_parsed = self.__parseThresholds__(thresholds, lastaccess) if is_parsed: self.__logger.info('Successfully parsed the thresholds: %s' % str(self.__thresholds)) else: self.__logger.error( 'Failed to parse thresholds from configuration file. Using default ones: %s' % str(self.__thresholds)) # parse algorithm2 parameters algorithm2_spacetokens = config.get_config('algorithm2_spacetokens', list) if algorithm2_spacetokens is None: algorithm2_spacetokens = [] algorithm2_creationlimit = config.get_dict('algorithm2_creationlimit', type='positive_int') self.__algorithm2_spacetokens, self.__algorithm2_creationlimit = self.__verifyAlgorithm2param( algorithm2_spacetokens, algorithm2_creationlimit) self.__logger.debug('Algorithm2_spacetokens : %s' % self.__algorithm2_spacetokens) self.__logger.debug('Algorithm2_creationlimit : %s' % self.__algorithm2_creationlimit) def __verifyAlgorithm2param(self, algorithm2_spacetokens, algorithm2_creationlimit): spacetokens = [] creationlimit = {} for token in algorithm2_spacetokens: if not token in algorithm2_creationlimit: self.__logger.warning( 'Algorithm2: %s is in spacetoken list, but with invalid or absent parameter info' % token) self.__logger.warning( 'Algorithm2: %s - using default algorithm instead' % token) else: spacetokens.append(token) creationlimit[token] = algorithm2_creationlimit[token] return spacetokens, creationlimit def __parseThresholds__(self, thresholds, lastaccess): if thresholds is None: return None try: threshold_list = [] for i in xrange(len(thresholds)): cur_threshold = { 'threshold': int(thresholds[i]), 'lastaccess': sorted(castListToIntegers(lastaccess[i]), reverse=True) } threshold_list.append(cur_threshold) except Exception: self.__logger.error( 'Failed to parse thresholds from configuration file') self.__logger.info(traceback.format_exc()) return None self.__thresholds = threshold_list return True def __appendDatasets(self, datastetsToCheck, oldDatasets): ''' this function filters out from datastetsToCheck the datasets, which are already present in oldDatasets and return a filtered list of datasets (newDatasets) ''' newDatasets = [] for data in datastetsToCheck: ######## check if dataset is already in the list if data[self.__DATASET_NAME_COL] not in oldDatasets: ######## check if one dataset is included few times in one set if data in newDatasets: text = 'Duplicate dataset:%s returned for site' % data[ self.__DATASET_NAME_COL] self.__logger.warning(text) continue newDatasets.append(data) return newDatasets def __filterDatasetsToDelete(self, site, spacetoclean, datasetsinfo): ''' It takes datasets from datasetsinfo and append it to deletion queue (datasetstodelete), while targetspace is not reached/ or the end of datasetsinfo is reached ''' datasetstodelete = [] spacefordataset = [] dates = [] naccs = [] cputimes = [] nBlocks = [] maxAccsCont = [] totalAccsCont = [] cleanedspace = 0 cleaned = False for data in datasetsinfo: cur = data[self.__DATASET_SIZE_COL] if not isinstance(cur, int): self.__logger.info( 'Dataset: %s has invalid DATASETSIZE attribute: %s' % (data[self.__DATASET_NAME_COL], str(cur))) self.__logger.info( 'Invoking alternative method to calculate dataset size...') cur = self.__getreplicasize(data[self.__DATASET_NAME_COL]) if not isinstance(cur, int): continue self.__logger.info('The size of dataset is: %d Bytes' % cur) cleanedspace += cur datasetstodelete += [data[self.__DATASET_NAME_COL]] dt = data[self.__REPLICA_CREATION_DATE_COL] dates += [epochTime(dt)] spacefordataset += [cur] if len(data) > self.__DATASET_SIZE_COL: naccs += [data[self.__NACC_COL]] cputimes += [data[self.__CPU_COL]] nBlocks += [data[self.__NBLOCKS_COL]] maxAccsCont += [data[self.__NMAX_COL]] totalAccsCont += [data[self.__NTOTAL_COL]] else: naccs += [None] cputimes += [None] nBlocks += [None] maxAccsCont += [None] totalAccsCont += [None] if cleanedspace >= spacetoclean: cleaned = True break return datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont def getSecondarySpace(self, site): return self.__getSpaceForDatasets(site, 0, 1e6, 0) def getOldSecondarySpace(self, site): return self.__getSpaceForDatasets(site, 0, 1e6, self.__creationlimit) def __getSpaceForDatasets(self, site, lastaccess, threshold, creationlimit): secondarydatasets = self.__popularity.getUnpopularDatasets( site, lastaccess, threshold, creationlimit) space = 0 for data in secondarydatasets: cur = data[self.__DATASET_SIZE_COL] if not isinstance(cur, int): self.__logger.info( 'Dataset: %s has invalid DATASETSIZE attribute: %s' % (data[self.__DATASET_NAME_COL], str(cur))) self.__logger.info( 'Invoking alternative method to calculate dataset size...') cur = self.__getreplicasize(data[self.__DATASET_NAME_COL]) if not isinstance(cur, int): continue self.__logger.info('The size of dataset is: %d Bytes' % cur) space += cur return space def __getDatasetsToDeleteAlgorithm2(self, site, spacetoclean, spacetoken): threshold = 1e12 lastaccess = 0 creationlimit = self.__algorithm2_creationlimit[spacetoken] #days replicatype = self.__algorithm2_custodiality #'default' datasets = self.__popularity.getUnpopularDatasets( site, lastaccess=lastaccess, threshold=threshold, creationlimit=creationlimit, replicatype=replicatype) datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__filterDatasetsToDelete( site, spacetoclean, datasets) return datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont def __getDatasetsToDeleteStandard(self, site, spacetoclean): cleaned = False totalcleaned = 0 datasetstodeleteTotal = [] spacefordatasetTotal = [] datesTotal = [] for threshold_set in self.__thresholds: threshold = threshold_set['threshold'] for lastaccess in threshold_set['lastaccess']: self.__logger.debug('Threshold: %d Lastaccess: %d' % (threshold, lastaccess)) datasets = self.__popularity.getUnpopularDatasets( site, lastaccess=lastaccess, threshold=threshold, creationlimit=self.__creationlimit) datasets = self.__appendDatasets(datasets, datasetstodeleteTotal) datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__filterDatasetsToDelete( site, spacetoclean, datasets) totalcleaned += cleanedspace datasetstodeleteTotal += datasetstodelete spacefordatasetTotal += spacefordataset datesTotal += dates if cleaned: break if cleaned: break return datasetstodeleteTotal, spacefordatasetTotal, datesTotal, cleaned, totalcleaned, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont def getDatasetsToDelete(self, site, spacetoclean, spacetoken): #Alternative algorithm for SCRATCHDISK if spacetoken in self.__algorithm2_spacetokens: datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__getDatasetsToDeleteAlgorithm2( site, spacetoclean, spacetoken) return datasetstodelete, spacefordataset, dates, cleaned, cputimes, naccs, cleanedspace, None, nBlocks, maxAccsCont, totalAccsCont #Default algorithm for DATADISK else: datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__getDatasetsToDeleteStandard( site, spacetoclean) if cleaned: return datasetstodelete, spacefordataset, dates, cleaned, cputimes, naccs, cleanedspace, None, nBlocks, maxAccsCont, totalAccsCont else: self.__logger.info( "Getting space for secondary datasets for site %s....." % site) totalscnd = self.getSecondarySpace(site) oldscnd = self.getOldSecondarySpace(site) scndspacesummary = [totalscnd, oldscnd] self.__logger.info( "Finished with secondary datasets site %s....." % site) return datasetstodelete, spacefordataset, dates, cleaned, cputimes, naccs, cleanedspace, scndspacesummary, nBlocks, maxAccsCont, totalAccsCont
class ReplicaReduction: __datasetSelection = DatasetSelection() __logger = logging.getLogger("dq2.victor.replicaReduction") __victorDao = VictorDao() def __prepareSummary(self, site, datasets, sizes, creationdates, cpus, naccs, spacesummary, cleaned, nBlocks, maxAccsCont, totalAccsCont): self.__logger.info('Going to create the cleaning summary for site %s' % (site)) dumpCleaningInfo(site, datasets, sizes, creationdates, cpus, naccs, spacesummary, cleaned, nBlocks, maxAccsCont, totalAccsCont) self.__victorDao.insertCleaningSummary(site, datasets, sizes, creationdates, cpus, naccs, spacesummary, cleaned, nBlocks, maxAccsCont, totalAccsCont) return def __clean(self, tobedeleted, refreshed): ''' @return: True: all necessary information available @return: False: if site is not allowed to be cleaned automatically => shifters should take care of this site ''' if (tobedeleted is None) and (not refreshed): return False return True def __afterAccountingSummary(self, fullSites): data = {} for site in fullSites: data[site] = { 'used_before': used, 'cleaned': spacetoclean, 'used_after': used - spacetoclean, 'total': total } dumpInfo(data, 'AccountingSummary_cleaning') return def cleanSites(self, fullSites, accountingSummary): datasets = {} successfullycleaned = {} markedfordeletion = {} indeletionqueue_dict = {} is_site_refreshed = {} datasets_per_site = {} for site in fullSites: ratio, used, total, tobedeleted, indeletionqueue, refreshed, spacetoclean, spacetoken = fullSites[ site] datasetstodelete, spacefordatasets, dates, cleaned, cpus, naccs, cleanedspace, scndspacesummary, nBlocks, maxAccsCont, totalAccsCont = self.__datasetSelection.getDatasetsToDelete( site, spacetoclean, spacetoken) if cleaned: self.__logger.info("Site %s can be successfully cleaned" % site) else: self.__logger.warning("Site %s could not be cleaned" % site) spacesummary = [used, total, cleanedspace] datasets_per_site[site] = [ datasetstodelete, spacefordatasets, dates, spacesummary, scndspacesummary, cpus, naccs, nBlocks, maxAccsCont, totalAccsCont ] markedfordeletion[site] = tobedeleted indeletionqueue_dict[site] = indeletionqueue successfullycleaned[site] = cleaned is_site_refreshed[site] = refreshed accountingSummary[site]['newlycleaned'] = cleanedspace #Mark datasets for deletion is_allowed_tobecleaned = {} self.__logger.info("Start marking dataset replicas for deletion...") for site in datasets_per_site: if not self.__clean(markedfordeletion[site], is_site_refreshed[site]): self.__logger.info( 'Site %s is not allowed to be cleaned. No cleaning done!' % site) #in debugging mode is_allowed_tobecleaned[site] = False continue is_allowed_tobecleaned[site] = True if not DEBUG_MODE: self.__replicadeletion.markReplicaForDeletion( site, datasets_per_site[site]) self.__logger.info( 'Deletion commented out: not marking datasets %s for site %s' % (site)) #in debugging mode else: self.__logger.info( 'Dummy mode: not marking datasets for deletion for site %s' % (site)) #in debugging mode self.__prepareSummary( site, datasets_per_site[site][0], datasets_per_site[site][1], datasets_per_site[site][2], datasets_per_site[site][5], datasets_per_site[site][6], datasets_per_site[site][3], successfullycleaned[site], datasets_per_site[site][7], datasets_per_site[site][8], datasets_per_site[site][9]) self.__logger.info("Finished marking datasets as ToBeDeleted") if not DEBUG_MODE: sendUncleanedMail(successfullycleaned) return datasets, successfullycleaned, accountingSummary