def __init__(self):
user = config.get_config('db_user', type='str')
password = config.get_config('db_password', type='str')
connectionName = config.get_config('db_conn', type='str')
if not user or not password or not connectionName:
raise Exception('No DB connection specified in configuration file')
connection_string = '%s/%s@%s'%(user, password, connectionName)
self.__connection = cx_Oracle.Connection(connection_string)
self.__logger = logging.getLogger('dq2.victor.victorDao')
def __init__(self):
user = config.get_config('db_user', type='str')
password = config.get_config('db_password', type='str')
connectionName = config.get_config('db_conn', type='str')
if not user or not password or not connectionName:
raise Exception('No DB connection specified in configuration file')
connection_string = '%s/%s@%s' % (user, password, connectionName)
self.__connection = cx_Oracle.Connection(connection_string)
self.__logger = logging.getLogger('dq2.victor.victorDao')
class Victor:
"""
Class for cleaning agent
It will monitor sites which have content over total space ratio above some given value
and clean dataset replicas from these overflowed sites according to algorithm provided in DatasetusageMonitir class
"""
__logger = logging.getLogger("dq2.victor")
__period = HOUR
__victorDao = VictorDao()
def __init__(self, name, configFile):
"""
Initializer for the object.
"""
self.__logger.info('Victor starting up...')
self.__siteSelection = SiteSelection()
self.__replicaReduction = ReplicaReduction()
def run(self):
try:
self.__victorDao.insertRun()
#Development: Uncomment later
fullSiteInfo, accountingSummary = self.__siteSelection.getFullSites()
#fullSiteInfo = {'SARA-MATRIX_DATADISK': (0.01, 990*(10**12), 1000*(10**12), 0, 0, True, 100**12, 'DATADISK')}
#fullSiteInfo = {u'T2_CH_CSCS^b-physics': (-0.19219963159204001, 119219963159204, 100000000000000.0, 0, 0, True, 49219963159204.008, None)} #accountingSummary = { u'T2_CH_CSCS^b-physics': {'total': 100000000000000.0, 'tobedeleted': 0, 'used': 119219963159204, 'indeletionqueue': 0, 'newlycleaned': 0}}
#fullSiteInfo = {'T2_US_Vanderbilt^heavy-ions': fullSiteInfo['T2_US_Vanderbilt^heavy-ions'], 'T2_BE_UCL^AnalysisOps': fullSiteInfo['T2_BE_UCL^AnalysisOps'],}
#accountingSummary = {'T2_US_Vanderbilt^heavy-ions': accountingSummary['T2_US_Vanderbilt^heavy-ions'], 'T2_BE_UCL^AnalysisOps': accountingSummary['T2_BE_UCL^AnalysisOps']}
datasetsForSite, successfullycleaned, accountingSummary = self.__replicaReduction.cleanSites(fullSiteInfo, accountingSummary)
prepareSummary(accountingSummary)
self.__victorDao.insertAccountingSummary(accountingSummary)
self.__victorDao.closeRun()
except Exception as e:
print(traceback.format_exc())
self.__logger.critical(traceback.format_exc())
def __init__ (self, query=False, remote=False):
"""
Class constructor.
@author: Miguel Branco
@contact: [email protected]
@since: 0.3
@version: $Id: DQDashboardTool.py,v 1.1 2008-05-29 11:15:39 psalgado Exp $
@param query: Boolean indicating if default query options are available
or not
@param remote: Boolean indicating if default remote access (service, port)
options are available or not
"""
self.__class__._logger = logging.getLogger(self.__module__)
self.parser = OptionParser(usage=self.usage, version=self.version,
description=self.description)
# default options are always available
options = self.defaultOptions
# if this is a 'query' tool, default query options are available
if query:
options.extend(self.defaultQueryOptions)
# if this is a 'remote access' tool, service, port and alike options
# are also available
if remote:
options.extend(self.remoteOptions)
# finally, tool specific options should be added
options.extend(self.toolOptions)
# add all the options to the parser
for option in options:
self.parser.add_option(option)
# parse the command line arguments and options and update the options
(self.options, self.args) = self.parser.parse_args()
class AccountingInterface(AccountingInterface):
"""
Class encapsulating the CMS Storage Information Interface.
"""
__freeRatio = {'default': 0.1}
__freeAbsolute = {'default': 25 * TERA}
__targetRatio = {'default': 0.2}
__targetAbsolute = {}
__specifiedSpacetokens = []
__centralPledge = 0.7
threshold_per_site = {}
__logger = logging.getLogger("dq2.victor.AccountingInterface")
#__configurationInterface='DQ2ConfigurationInterface'
def __init__(self, name):
self.__nodeusage = self.__get_nodeusage_values()
self.__groupusage = self.__get_groupusage_values()
self.__logger.debug('NODE USAGE: %s' % self.__nodeusage)
self.__logger.debug('GROUP USAGE: %s' % self.__groupusage)
freeRatio = config.get_dict('freeRatio', type=float)
if freeRatio is not None:
self.__freeRatio = freeRatio
freeAbsolute = config.get_dict('freeAbsolute', type='size')
if freeAbsolute is not None:
self.__freeAbsolute = freeAbsolute
targetRatio = config.get_dict('targetRatio', type=float)
if targetRatio is not None:
self.__targetRatio = targetRatio
targetAbsolute = config.get_dict('targetAbsolute', type='size')
if targetAbsolute is not None:
self.__targetAbsolute = targetAbsolute
centralPledge = config.get_dict('centralPledge', type=float)
if centralPledge is not None:
self.__centralPledge = centralPledge
self.__site_name_map = self.__get_site_name_map()
self.__sitedb_pledges = self.__get_sitedb_pledges()
def __get_site_name_map(self):
url = 'https://cmsweb.cern.ch/sitedb/data/prod/site-names'
site_map_data = get_json_data_https(url)
siteIdx = site_map_data['desc']['columns'].index('site_name')
typeIdx = site_map_data['desc']['columns'].index('type')
nameIdx = site_map_data['desc']['columns'].index('alias')
site_name_map = {}
for item in site_map_data['result']:
try:
(site_name_map[item[siteIdx]][item[typeIdx]]).append(
item[nameIdx])
except KeyError:
try:
site_name_map[item[siteIdx]][item[typeIdx]] = [
item[nameIdx]
]
except KeyError:
site_name_map[item[siteIdx]] = {
item[typeIdx]: [item[nameIdx]]
}
self.__logger.debug(site_name_map)
return site_name_map
def __get_sitedb_pledges(self):
url = 'https://cmsweb.cern.ch/sitedb/data/prod/resource-pledges'
resource_pledge_data = get_json_data_https(url)
siteIdx = resource_pledge_data['desc']['columns'].index('site_name')
updateTimeIdx = resource_pledge_data['desc']['columns'].index(
'pledge_date')
yearIdx = resource_pledge_data['desc']['columns'].index('quarter')
diskIdx = resource_pledge_data['desc']['columns'].index('disk_store')
localDiskIdx = resource_pledge_data['desc']['columns'].index(
'local_store')
resource_pledges = {}
current_year = time.gmtime().tm_year
for item in resource_pledge_data['result']:
if item[yearIdx] == current_year:
try:
if item[updateTimeIdx] >= resource_pledges[
item[siteIdx]]['pledge_date']:
resource_pledges[
item[siteIdx]]['disk_store'] = item[diskIdx]
resource_pledges[
item[siteIdx]]['local_store'] = item[localDiskIdx]
resource_pledges[
item[siteIdx]]['pledge_date'] = item[updateTimeIdx]
except KeyError:
resource_pledges[item[siteIdx]] = {
'disk_store': item[diskIdx],
'local_store': item[localDiskIdx],
'pledge_date': item[updateTimeIdx]
}
self.__logger.debug(resource_pledges)
return resource_pledges
def __get_nodeusage_values(self):
url = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/nodeusage"
nodeusage_aux = get_json_data(url)
nodeusage = {}
#Massage information to make it more accessible
for site_dic in nodeusage_aux['phedex']['node']:
site = site_dic["name"]
try:
nodeusage[site] = {
"src_node_bytes": int(site_dic["src_node_bytes"]),
"nonsrc_node_bytes": int(site_dic["nonsrc_node_bytes"]),
"noncust_dest_bytes": int(site_dic["noncust_dest_bytes"]),
"noncust_node_bytes": int(site_dic["noncust_node_bytes"]),
"cust_node_bytes": int(site_dic["cust_node_bytes"]),
"cust_dest_bytes": int(site_dic["cust_dest_bytes"])
}
except:
self.__logger.error(
'Incomplete information for site %s in %s' % (site, url))
return nodeusage
def __get_groupusage_values(self):
url = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/groupusage"
groupusage_aux = get_json_data(url)
groupusage = {}
#Massage information to make it more accessible
for site_dic in groupusage_aux['phedex']['node']:
site = site_dic["name"]
groupusage[site] = {}
for group_dic in site_dic['group']:
group = group_dic["name"]
try:
groupusage[site][group] = {
"dest_bytes": int(group_dic["dest_bytes"]),
"node_bytes": int(group_dic["node_bytes"])
}
except:
self.__logger.error(
'Incomplete information for group %s in site %s in %s'
% (group, site, url))
return groupusage
def calculateSpaceToClean(self, site, used, total, tobedeleted,
indeletionqueue):
site, physicsgroup = site.split('^')
targetAbsolute = 25 * TERA
targetRatio = 0.3
bigsiteThreshold = 100 * TERA
if total > bigsiteThreshold:
targetValue = total - targetAbsolute #Targetvalue is of used diskspace
else:
targetValue = total * (1 - targetRatio
) #Targetvalue is of used diskspace
effectivelyused = used - tobedeleted - indeletionqueue
tobecleaned = effectivelyused - targetValue
return tobecleaned, None
def getSites(self, spacetokens=[]):
"""
@return: List with all the sites.
"""
sites = []
for site in self.__site_name_map:
site_name = self.__site_name_map[site]['cms'][0]
try:
groups = groupPledges[site_name].keys()
for group in groups:
sites.append('%s^%s' % (site_name, group))
except KeyError:
pass
return sites
def __getUsedSpaceCentral(self, site):
try:
#Basic usage of storage
bytes = self.__nodeusage[site][
'noncust_node_bytes'] + self.__nodeusage[site][
'src_node_bytes'] + self.__nodeusage[site][
'nonsrc_node_bytes']
#Add reserved space of ongoing transfers
#bytes = bytes + self.__nodeusage[site]['noncust_dest_bytes']
#THE PREVIOUS CASE IS ONLY VALID FOR T2s!!!
self.__logger.info('Used space for site %s: %.2f' %
(site, bytes / TERA))
return bytes
except Exception as e:
self.__logger.error('No used space for site %s (%s)' % (site, e))
return None
def __getUsedSpaceGroup(self, site, group):
try:
#Basic usage of storage
bytes = self.__groupusage[site][group]['node_bytes']
self.__logger.info('Used space for site %s by group %s: %.2f' %
(site, group, bytes / TERA))
return bytes
except Exception as e:
self.__logger.error('No used space for site %s by group %s (%s)' %
(site, group, e))
return None
def getUsedSpace(self, site):
site, physicsgroup = site.split('^')
return self.__getUsedSpaceGroup(site, physicsgroup)
def __getSiteDBPledge(self, site):
for item in self.__site_name_map:
if self.__site_name_map[item]['cms'][0] == site:
siteId = item
try:
pledge = ((self.__sitedb_pledges[siteId]['disk_store'] or 0) -
(self.__sitedb_pledges[siteId]['local_store']
or 0)) * TERA * self.__centralPledge
if pledge is not None:
self.__logger.info('Pledged space for site %s: %.2f' %
(site, pledge / TERA))
return pledge
except KeyError:
self.__logger.error('No pledge for site %s' % (site))
return None
def __getGroupPledge(self, site, group):
try:
bytes = groupPledges[site][group]
self.__logger.info('Pledged space for site %s by group %s: %.2f' %
(site, group, bytes / TERA))
return bytes
except Exception as e:
self.__logger.error(
'No pledged space for site %s by group %s (%s)' %
(site, group, e))
return None
def getTotalSpace(self, site):
site, physicsgroup = site.split('^')
return self.__getGroupPledge(site, physicsgroup)
def getToBeDeletedSpace(self, site):
"""
MOCK. Not needed by CMS
"""
return 0
def refreshSiteStatistics(self, site):
"""
MOCK. Not needed by CMS
"""
return True
def getSpaceInDeletionQueue(self, site):
'''
Return space which is freed up by deletion
'''
return 0
#try:
#Basic usage of storage
# bytes = self.__nodeusage[site]['src_node_bytes'] + self.__nodeusage[site]['nonsrc_node_bytes']
# self.__logger.info('Space in deletion queue for site %s: %.2f' %(site, bytes/TERA))
# return bytes
#except:
# self.__logger.error('No space in deletion queue for site %s' %(site))
# return 0 #It has to be 0, not None!
def isFull(self, site, ratio, free):
thresholdFree = 0.1
thresholdAbsFree = 15 * TERA
if ratio > thresholdFree or free > thresholdAbsFree:
return False
return True
class PopularityInterface(PopularityInterface):
"""
Class encapsulating the CMS Popularity Interface.
"""
__logger = logging.getLogger("dq2.victor.PopularityInterface")
__MONTH = 30
__REPLICA_CREATION_DATE_COL = 4
def __init__(self, name):
#Create a temporary directory to cache information
self.__tmpdirectory = tempfile.mkdtemp()
def __hasCustodialCopy(self, blockname):
url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/blockreplicas?custodial=y&complete=y&block=%s' % (
blockname)
url = url.replace('#', '%23')
self.__logger.debug(url)
replicas = get_json_data(url)
'''
replicas = {
"phedex":{
"request_version":"2.1.8",
"request_timestamp":1311755259.31146,
"instance":"prod",
"request_call":"blockreplicas",
"request_url":"http://cmsweb.cern.ch:7001/phedex/datasvc/json/prod/blockreplicas",
"request_date":"2011-07-27 08:27:39 UTC",
"block":[
{
"bytes":"67095800951",
"files":"32",
"is_open":"n",
"name":"/ZJetsToNuNu_Pt-100_7TeV-herwigpp/Summer11-START311_V2-v1/GEN-SIM#7f6b861b-2263-4854-8abf-d096d35d9f1a",
"id":"2576551",
"replica":[
{
"bytes":"67095800951",
"files":"32",
"node":"T1_IT_CNAF_MSS",
"time_create":"1311331011",
"time_update":"1311610457.398",
"group":"DataOps",
"node_id":"8",
"custodial":"y",
"se":"storm-fe-cms.cr.cnaf.infn.it",
"subscribed":"y",
"complete":"y"}]}],
"call_time":"0.05906"}}
'''
try:
if replicas['phedex']['block'][0]['replica']:
return True
except KeyError:
self.__logger.warning(
'Block %s excepted with KeyError. replicas = %s' %
(blockname, replicas))
return False
except IndexError:
self.__logger.warning(
'Block %s excepted with IndexError. replicas = %s' %
(blockname, replicas))
return False
return False
def __getDatasetStats(self, blocks):
datasetStats = {}
for blockname in blocks:
datasetName = blockname.split('#')[0]
datasetStats.setdefault(datasetName, {
'max': -1,
'sum': 0,
'nblocks': 0
})
datasetStats[datasetName]['nblocks'] += 1
datasetStats[datasetName]['sum'] += int(
blocks[blockname]['popularitynacc'])
if datasetStats[datasetName]['max'] < int(
blocks[blockname]['popularitynacc']):
datasetStats[datasetName]['max'] = int(
blocks[blockname]['popularitynacc'])
return datasetStats
def __validateBlock(self, blockname, blockinfo, group, maxnacc, maxcdate):
if blockinfo['custodial'] != 'n':
#self.__logger.debug('Refuse because CUSTODIAL')
return False
if blockinfo['group'] != group:
#self.__logger.debug('Refuse because we look for replicas from %s and the replica belongs to %s' %(group, blockinfo['group']))
return False
if blockinfo['popularitynacc'] > maxnacc:
self.__logger.debug(
'Refuse because replica has been accessed %s times (>%s)' %
(blockinfo['popularitynacc'], maxnacc))
return False
cdate = datetime.datetime.fromtimestamp(blockinfo['creation_time'])
cdate_max = datetime.datetime.now() - datetime.timedelta(days=maxcdate)
if cdate > cdate_max:
self.__logger.debug(
'Refuse because replica is too young %s (>%s)' %
(cdate, cdate_max))
return False
if not self.__hasCustodialCopy(blockname):
self.__logger.debug('Refuse because replica has no custodial copy')
return False
return True
def __parseUnpopularBlocks(self, site, unpopularBlocks, threshold,
creationlimit, physicsgroup):
startdate = unpopularBlocks['popularitytstart']
enddate = unpopularBlocks['popularitytstop']
blocks = unpopularBlocks[site]
blocks_list = []
datasetStats = self.__getDatasetStats(blocks)
for blockname in blocks:
datasetname = blockname.split('#')[0]
#Skip custodial blocks, blocks that don't belong to central AnalysisOps and blocks without a custodial copy
#if blocks[blockname]['custodial']=='n' and blocks[blockname]['group']=='AnalysisOps' and self.__hasCustodialCopy(blockname):
if self.__validateBlock(blockname, blocks[blockname], physicsgroup,
threshold, creationlimit):
try:
new_block = (str(blockname),
int(blocks[blockname]['popularitynacc']),
None,
datetime.datetime.fromtimestamp(
blocks[blockname]['creation_time']),
datetime.datetime.fromtimestamp(
blocks[blockname]['creation_time']),
int(blocks[blockname]['size']),
int(blocks[blockname]['popularitycpu']),
int(blocks[blockname]['popularitynacc']),
datasetStats[datasetname]['nblocks'],
datasetStats[datasetname]['max'],
datasetStats[datasetname]['sum'])
blocks_list.append(new_block)
self.__logger.debug('Appended %s to %s: %s' %
(blockname, site, new_block))
except:
self.__logger.error(
'Error processing block %s for site %s. Complete information: %s'
% (blockname, site, blocks[blockname]))
continue
else:
self.__logger.debug('Refused %s for %s' % (blockname, site))
#self.__logger.debug('Block %s skipped: custodial %s and group %s' %(blockname, blocks[blockname]['custodial'], blocks[blockname]['group']))
return blocks_list
def __getBlocks(self, site):
if os.path.isfile('%s/blocks_%s' % (self.__tmpdirectory, site)):
unpopularBlocks = get_json_data_from_file(
'%s/blocks_%s' % (self.__tmpdirectory, site))
return unpopularBlocks
else:
url = 'https://cmsweb.cern.ch/popdb/victorinterface/popdbcombine/?sitename=%s' % (
site)
unpopularBlocks = get_json_data_https(url)
dumpTemporaryInfo(unpopularBlocks, self.__tmpdirectory,
'blocks_%s' % (site))
return unpopularBlocks
def getUnpopularDatasets(self,
site,
lastaccess,
threshold,
creationlimit,
replicatype='secondary'):
'''
Get the list of a site's unpopular block replicas according to the input criteria.
'''
def compdate(x, y):
COMPARE_COL = self.__REPLICA_CREATION_DATE_COL
if epochTime(x[COMPARE_COL]) > epochTime(y[COMPARE_COL]):
return 1
elif epochTime(x[COMPARE_COL]) < epochTime(y[COMPARE_COL]):
return -1
else:
return 0
site, physicsgroup = site.split('^')
st = time.time()
self.__logger.info('Getting unpopular datasets for %s...' % site)
try:
blocks = self.__getBlocks(site)
if not blocks or not blocks[site]:
raise Exception("Empty answer received from unpopularity API")
unpopularBlocksListed = self.__parseUnpopularBlocks(
site, blocks, threshold, creationlimit, physicsgroup)
except Exception as e:
self.__logger.critical('Failed to process site %s [%s - %s]' %
(site, e, traceback.format_exc()))
sendErrorMail('%s\n%s' % (e, traceback.format_exc()))
return {}
finally:
elapsed = round(time.time() - st, 2)
self.__logger.info(
'...back from popDB API for site %s [Took %.2f seconds]' %
(site, elapsed))
unpopularBlocksListed = sorted(unpopularBlocksListed, compdate)
return unpopularBlocksListed
class SiteSelection:
__logger = logging.getLogger("dq2.victor.siteSelection")
__accountingInterface = 'DQ2AccountingInterface'
def __init__(self):
accountingInterface = config.get_config('accountingInterface',
type=str)
if accountingInterface:
self.__accountingInterface = accountingInterface
self.__accounting = create_tool(self.__accountingInterface)
def getFullSites(self):
sitestoinspect = self.__accounting.getSites()
fullSiteInfo = {}
allInfo = {}
self.__logger.info('Sites to check for full storages: %s' %
sitestoinspect)
#sitestoinspect = sitestoinspect[0:5]
for site in sitestoinspect:
self.__logger.debug('Going to evaluate site %s' % (site))
refreshed = self.__accounting.refreshSiteStatistics(site)
used = self.__accounting.getUsedSpace(site)
total = self.__accounting.getTotalSpace(site)
tobedeleted = self.__accounting.getToBeDeletedSpace(site)
indeletionqueue = self.__accounting.getSpaceInDeletionQueue(site)
allInfo[site] = {
'used': used,
'total': total,
'tobedeleted': tobedeleted,
'indeletionqueue': indeletionqueue,
'newlycleaned': 0
}
if used == None or total == None or total == 0:
self.__logger.warning(
'Skipped site: %s - Missing values for used and/or pledged space'
% (site))
continue
free = total - used + tobedeleted + indeletionqueue
ratio = free * 1.0 / total
self.__logger.info(
'Accounting Summary. Site: %s - Used: %.2fTB Free: %.2fTB (%.2f) Total: %.2fTB ToBeDeleted: %.2fTB InDeletionQueue %.2fTB RefreshedStatistics: %s'
% (site, used / TERA, free / TERA, ratio, total / TERA,
tobedeleted / TERA, indeletionqueue / TERA, refreshed))
if not self.__accounting.isFull(site, ratio, free):
self.__logger.debug('Site %s is not full' % (site))
continue
spacetoclean, spacetoken = self.__accounting.calculateSpaceToClean(
site, used, total, tobedeleted, indeletionqueue)
fullSiteInfo[site] = (ratio, used, total, tobedeleted,
indeletionqueue, refreshed, spacetoclean,
spacetoken)
thresholdInfo = self.__accounting.threshold_per_site
return fullSiteInfo, allInfo
"""
@copyright: European Organization for Nuclear Research (CERN)
@author: Andrii Thykonov U{[email protected]<mailto:[email protected]>}, CERN, 2010-2011
@author: Fernando H. Barreiro U{[email protected]<mailto:[email protected]>}, CERN, 2011
@license: Licensed under the Apache License, Version 2.0 (the "License");
You may not use this file except in compliance with the License.
You may obtain a copy of the License at U{http://www.apache.org/licenses/LICENSE-2.0}
"""
from dq2.common import log as logging
from dq2.victor import config
import urllib2, httplib
import simplejson
logger = logging.getLogger("dq2.victor.utils")
class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
"""
Simple HTTPS client authentication class based on provided
key/ca information
"""
def __init__(self, key=None, cert=None, level=0):
if level > 1:
urllib2.HTTPSHandler.__init__(self, debuglevel=1)
else:
urllib2.HTTPSHandler.__init__(self)
self.key = key
self.cert = cert
class Timer:
"""
Class with Borg design pattern:
http://code.activestate.com/recipes/66531-singleton-we-dont-need-no-stinkin-singleton-the-bo/
The dictionary
"""
__timings = {
'accounting': {
'calls': 0,
'time': 0
},
'deletedVol': {
'calls': 0,
'time': 0
},
'refreshAcc': {
'calls': 0,
'time': 0
},
'popularity': {
'calls': 0,
'time': 0
},
'setMetadata': {
'calls': 0,
'time': 0
},
'thresholds': {
'calls': 0,
'time': 0
}
}
__logger = logging.getLogger("dq2.victor.timer")
def __init__(self):
self.__dict__ = self.__shared_state
def increaseAccounting(self, time):
self.__timings['accounting']['calls'] += 1
self.__timings['accounting']['time'] += time
def increaseDeletedVolume(self, time):
self.__timings['deletedVol']['calls'] += 1
self.__timings['deletedVol']['time'] += time
def increasePopularity(self, time):
self.__timings['popularity']['calls'] += 1
self.__timings['popularity']['time'] += time
def increaseSetMetadata(self, time):
self.__timings['setMetadata']['calls'] += 1
self.__timings['setMetadata']['time'] += time
def increaseThresholds(self, time):
self.__timings['thresholds']['calls'] += 1
self.__timings['thresholds']['time'] += time
def printout(self):
self.__logger.info("""
Accounting calls: %d in %.1f
DeletedVol calls: %d in %.1f
RefreshAcc calls: %d in %.1f
Popularity calls: %d in %.1f
setMetadata calls: %d in %.1f
Thresholds calls: %d in %.1f
""" % (self.__timings['accounting']['calls'],
self.__timings['accounting']['time'],
self.__timings['deletedVol']['calls'],
self.__timings['deletedVol']['time'],
self.__timings['refreshAcc']['calls'],
self.__timings['refreshAcc']['time'],
self.__timings['popularity']['calls'],
self.__timings['popularity']['time'],
self.__timings['setMetadata']['calls'],
self.__timings['setMetadata']['time'],
self.__timings['thresholds']['calls'],
self.__timings['thresholds']['time']))
class DatasetSelection:
__logger = logging.getLogger("dq2.victor.datasetSelection")
__popularityInterface = 'DQ2PopularityInterface'
MONTH = 30
# Algorithm 1. Default cleaning algorithm
#__thresholds = [{'threshold' : 1, 'lastaccess' : [3, 2, 1]}]
__thresholds = [{'threshold': 1, 'lastaccess': [1]}]
__creationlimit = 90 #days
# Algorithm 2. User spacetokens and similar. Delete by replica creation date
__algorithm2_custodiality = 'default'
__algorithm2_spacetokens = [] #'SCRATCHDISK']
__algorithm2_creationlimit = {} #{'SCRATCHDISK':15, 'PRODDISK':31}
#Positions of
__DATASET_NAME_COL = 0
__LASTACCESS_COL = 2
__DATASET_CREATION_DATE_COL = 3
__REPLICA_CREATION_DATE_COL = 4
__DATASET_SIZE_COL = 5
__CPU_COL = 6
__NACC_COL = 7
__NBLOCKS_COL = 8
__NMAX_COL = 9
__NTOTAL_COL = 10
def __init__(self):
popularityInterface = config.get_config('popularityInterface',
type=str)
if popularityInterface:
self.__popularityInterface = popularityInterface
self.__popularity = create_tool(self.__popularityInterface)
creationlimit = config.get_config('creationlimit', int)
if creationlimit is not None:
self.__creationlimit = creationlimit
thresholds = config.get_config('thresholds', list)
lastaccess = config.get_config('lastaccess', 'list_2')
is_parsed = self.__parseThresholds__(thresholds, lastaccess)
if is_parsed:
self.__logger.info('Successfully parsed the thresholds: %s' %
str(self.__thresholds))
else:
self.__logger.error(
'Failed to parse thresholds from configuration file. Using default ones: %s'
% str(self.__thresholds))
# parse algorithm2 parameters
algorithm2_spacetokens = config.get_config('algorithm2_spacetokens',
list)
if algorithm2_spacetokens is None:
algorithm2_spacetokens = []
algorithm2_creationlimit = config.get_dict('algorithm2_creationlimit',
type='positive_int')
self.__algorithm2_spacetokens, self.__algorithm2_creationlimit = self.__verifyAlgorithm2param(
algorithm2_spacetokens, algorithm2_creationlimit)
self.__logger.debug('Algorithm2_spacetokens : %s' %
self.__algorithm2_spacetokens)
self.__logger.debug('Algorithm2_creationlimit : %s' %
self.__algorithm2_creationlimit)
def __verifyAlgorithm2param(self, algorithm2_spacetokens,
algorithm2_creationlimit):
spacetokens = []
creationlimit = {}
for token in algorithm2_spacetokens:
if not token in algorithm2_creationlimit:
self.__logger.warning(
'Algorithm2: %s is in spacetoken list, but with invalid or absent parameter info'
% token)
self.__logger.warning(
'Algorithm2: %s - using default algorithm instead' % token)
else:
spacetokens.append(token)
creationlimit[token] = algorithm2_creationlimit[token]
return spacetokens, creationlimit
def __parseThresholds__(self, thresholds, lastaccess):
if thresholds is None:
return None
try:
threshold_list = []
for i in xrange(len(thresholds)):
cur_threshold = {
'threshold':
int(thresholds[i]),
'lastaccess':
sorted(castListToIntegers(lastaccess[i]), reverse=True)
}
threshold_list.append(cur_threshold)
except Exception:
self.__logger.error(
'Failed to parse thresholds from configuration file')
self.__logger.info(traceback.format_exc())
return None
self.__thresholds = threshold_list
return True
def __appendDatasets(self, datastetsToCheck, oldDatasets):
'''
this function filters out from datastetsToCheck the datasets,
which are already present in oldDatasets
and return a filtered list of datasets (newDatasets)
'''
newDatasets = []
for data in datastetsToCheck:
######## check if dataset is already in the list
if data[self.__DATASET_NAME_COL] not in oldDatasets:
######## check if one dataset is included few times in one set
if data in newDatasets:
text = 'Duplicate dataset:%s returned for site' % data[
self.__DATASET_NAME_COL]
self.__logger.warning(text)
continue
newDatasets.append(data)
return newDatasets
def __filterDatasetsToDelete(self, site, spacetoclean, datasetsinfo):
'''
It takes datasets from datasetsinfo and append it to deletion queue (datasetstodelete),
while targetspace is not reached/ or the end of datasetsinfo is reached
'''
datasetstodelete = []
spacefordataset = []
dates = []
naccs = []
cputimes = []
nBlocks = []
maxAccsCont = []
totalAccsCont = []
cleanedspace = 0
cleaned = False
for data in datasetsinfo:
cur = data[self.__DATASET_SIZE_COL]
if not isinstance(cur, int):
self.__logger.info(
'Dataset: %s has invalid DATASETSIZE attribute: %s' %
(data[self.__DATASET_NAME_COL], str(cur)))
self.__logger.info(
'Invoking alternative method to calculate dataset size...')
cur = self.__getreplicasize(data[self.__DATASET_NAME_COL])
if not isinstance(cur, int):
continue
self.__logger.info('The size of dataset is: %d Bytes' % cur)
cleanedspace += cur
datasetstodelete += [data[self.__DATASET_NAME_COL]]
dt = data[self.__REPLICA_CREATION_DATE_COL]
dates += [epochTime(dt)]
spacefordataset += [cur]
if len(data) > self.__DATASET_SIZE_COL:
naccs += [data[self.__NACC_COL]]
cputimes += [data[self.__CPU_COL]]
nBlocks += [data[self.__NBLOCKS_COL]]
maxAccsCont += [data[self.__NMAX_COL]]
totalAccsCont += [data[self.__NTOTAL_COL]]
else:
naccs += [None]
cputimes += [None]
nBlocks += [None]
maxAccsCont += [None]
totalAccsCont += [None]
if cleanedspace >= spacetoclean:
cleaned = True
break
return datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont
def getSecondarySpace(self, site):
return self.__getSpaceForDatasets(site, 0, 1e6, 0)
def getOldSecondarySpace(self, site):
return self.__getSpaceForDatasets(site, 0, 1e6, self.__creationlimit)
def __getSpaceForDatasets(self, site, lastaccess, threshold,
creationlimit):
secondarydatasets = self.__popularity.getUnpopularDatasets(
site, lastaccess, threshold, creationlimit)
space = 0
for data in secondarydatasets:
cur = data[self.__DATASET_SIZE_COL]
if not isinstance(cur, int):
self.__logger.info(
'Dataset: %s has invalid DATASETSIZE attribute: %s' %
(data[self.__DATASET_NAME_COL], str(cur)))
self.__logger.info(
'Invoking alternative method to calculate dataset size...')
cur = self.__getreplicasize(data[self.__DATASET_NAME_COL])
if not isinstance(cur, int):
continue
self.__logger.info('The size of dataset is: %d Bytes' % cur)
space += cur
return space
def __getDatasetsToDeleteAlgorithm2(self, site, spacetoclean, spacetoken):
threshold = 1e12
lastaccess = 0
creationlimit = self.__algorithm2_creationlimit[spacetoken] #days
replicatype = self.__algorithm2_custodiality #'default'
datasets = self.__popularity.getUnpopularDatasets(
site,
lastaccess=lastaccess,
threshold=threshold,
creationlimit=creationlimit,
replicatype=replicatype)
datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__filterDatasetsToDelete(
site, spacetoclean, datasets)
return datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont
def __getDatasetsToDeleteStandard(self, site, spacetoclean):
cleaned = False
totalcleaned = 0
datasetstodeleteTotal = []
spacefordatasetTotal = []
datesTotal = []
for threshold_set in self.__thresholds:
threshold = threshold_set['threshold']
for lastaccess in threshold_set['lastaccess']:
self.__logger.debug('Threshold: %d Lastaccess: %d' %
(threshold, lastaccess))
datasets = self.__popularity.getUnpopularDatasets(
site,
lastaccess=lastaccess,
threshold=threshold,
creationlimit=self.__creationlimit)
datasets = self.__appendDatasets(datasets,
datasetstodeleteTotal)
datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__filterDatasetsToDelete(
site, spacetoclean, datasets)
totalcleaned += cleanedspace
datasetstodeleteTotal += datasetstodelete
spacefordatasetTotal += spacefordataset
datesTotal += dates
if cleaned:
break
if cleaned:
break
return datasetstodeleteTotal, spacefordatasetTotal, datesTotal, cleaned, totalcleaned, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont
def getDatasetsToDelete(self, site, spacetoclean, spacetoken):
#Alternative algorithm for SCRATCHDISK
if spacetoken in self.__algorithm2_spacetokens:
datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__getDatasetsToDeleteAlgorithm2(
site, spacetoclean, spacetoken)
return datasetstodelete, spacefordataset, dates, cleaned, cputimes, naccs, cleanedspace, None, nBlocks, maxAccsCont, totalAccsCont
#Default algorithm for DATADISK
else:
datasetstodelete, spacefordataset, dates, cleaned, cleanedspace, cputimes, naccs, nBlocks, maxAccsCont, totalAccsCont = self.__getDatasetsToDeleteStandard(
site, spacetoclean)
if cleaned:
return datasetstodelete, spacefordataset, dates, cleaned, cputimes, naccs, cleanedspace, None, nBlocks, maxAccsCont, totalAccsCont
else:
self.__logger.info(
"Getting space for secondary datasets for site %s....." %
site)
totalscnd = self.getSecondarySpace(site)
oldscnd = self.getOldSecondarySpace(site)
scndspacesummary = [totalscnd, oldscnd]
self.__logger.info(
"Finished with secondary datasets site %s....." % site)
return datasetstodelete, spacefordataset, dates, cleaned, cputimes, naccs, cleanedspace, scndspacesummary, nBlocks, maxAccsCont, totalAccsCont
class ReplicaReduction:
__datasetSelection = DatasetSelection()
__logger = logging.getLogger("dq2.victor.replicaReduction")
__victorDao = VictorDao()
def __prepareSummary(self, site, datasets, sizes, creationdates, cpus,
naccs, spacesummary, cleaned, nBlocks, maxAccsCont,
totalAccsCont):
self.__logger.info('Going to create the cleaning summary for site %s' %
(site))
dumpCleaningInfo(site, datasets, sizes, creationdates, cpus, naccs,
spacesummary, cleaned, nBlocks, maxAccsCont,
totalAccsCont)
self.__victorDao.insertCleaningSummary(site, datasets, sizes,
creationdates, cpus, naccs,
spacesummary, cleaned, nBlocks,
maxAccsCont, totalAccsCont)
return
def __clean(self, tobedeleted, refreshed):
'''
@return: True: all necessary information available
@return: False: if site is not allowed to be cleaned automatically => shifters should take care of this site
'''
if (tobedeleted is None) and (not refreshed):
return False
return True
def __afterAccountingSummary(self, fullSites):
data = {}
for site in fullSites:
data[site] = {
'used_before': used,
'cleaned': spacetoclean,
'used_after': used - spacetoclean,
'total': total
}
dumpInfo(data, 'AccountingSummary_cleaning')
return
def cleanSites(self, fullSites, accountingSummary):
datasets = {}
successfullycleaned = {}
markedfordeletion = {}
indeletionqueue_dict = {}
is_site_refreshed = {}
datasets_per_site = {}
for site in fullSites:
ratio, used, total, tobedeleted, indeletionqueue, refreshed, spacetoclean, spacetoken = fullSites[
site]
datasetstodelete, spacefordatasets, dates, cleaned, cpus, naccs, cleanedspace, scndspacesummary, nBlocks, maxAccsCont, totalAccsCont = self.__datasetSelection.getDatasetsToDelete(
site, spacetoclean, spacetoken)
if cleaned:
self.__logger.info("Site %s can be successfully cleaned" %
site)
else:
self.__logger.warning("Site %s could not be cleaned" % site)
spacesummary = [used, total, cleanedspace]
datasets_per_site[site] = [
datasetstodelete, spacefordatasets, dates, spacesummary,
scndspacesummary, cpus, naccs, nBlocks, maxAccsCont,
totalAccsCont
]
markedfordeletion[site] = tobedeleted
indeletionqueue_dict[site] = indeletionqueue
successfullycleaned[site] = cleaned
is_site_refreshed[site] = refreshed
accountingSummary[site]['newlycleaned'] = cleanedspace
#Mark datasets for deletion
is_allowed_tobecleaned = {}
self.__logger.info("Start marking dataset replicas for deletion...")
for site in datasets_per_site:
if not self.__clean(markedfordeletion[site],
is_site_refreshed[site]):
self.__logger.info(
'Site %s is not allowed to be cleaned. No cleaning done!' %
site) #in debugging mode
is_allowed_tobecleaned[site] = False
continue
is_allowed_tobecleaned[site] = True
if not DEBUG_MODE:
self.__replicadeletion.markReplicaForDeletion(
site, datasets_per_site[site])
self.__logger.info(
'Deletion commented out: not marking datasets %s for site %s'
% (site)) #in debugging mode
else:
self.__logger.info(
'Dummy mode: not marking datasets for deletion for site %s'
% (site)) #in debugging mode
self.__prepareSummary(
site, datasets_per_site[site][0], datasets_per_site[site][1],
datasets_per_site[site][2], datasets_per_site[site][5],
datasets_per_site[site][6], datasets_per_site[site][3],
successfullycleaned[site], datasets_per_site[site][7],
datasets_per_site[site][8], datasets_per_site[site][9])
self.__logger.info("Finished marking datasets as ToBeDeleted")
if not DEBUG_MODE:
sendUncleanedMail(successfullycleaned)
return datasets, successfullycleaned, accountingSummary
