def convertQueueNameToDQ2Names(self, queue, test):
from pandatools import Client
from dq2.info.TiersOfATLAS import ToACache
sites = []
if 'PFT' in test.template.description:
#Client.PandaSites = Client.getSiteSpecs('production')[1]
Client.PandaSites = Client.getSiteSpecs('all')[1]
for site in Client.PandaSites[queue]['setokens'].values():
sites.append(Client.convSrmV2ID(site))
allowed_sites = []
for site in ToACache.sites:
if (site not in allowed_sites
and Client.convSrmV2ID(site) in sites
and site.find('TAPE') == -1 and 'DISK' in site):
allowed_sites.append(site)
return allowed_sites
def change_site_status(self, site, new_status):
if new_status not in ('test', 'online'):
return False
self.add_log('Changing %s status to %s:' % (site, new_status))
if self.debug:
self.add_log('DEBUG mode')
return True
now = int(time.time())
if new_status == 'test':
try:
last_exclusion = int(Site.objects.filter(name=site)[0].getSiteOptions_for_site.filter(option_name='last_exclusion')[0].option_value)
except:
last_exclusion = 0
if now - last_exclusion < 21600:
self.add_log('%s was recently auto-excluded. Skipping...' % site)
return False
cmd = "curl -s -k --cert $X509_USER_PROXY 'https://panda.cern.ch:25943/server/controller/query?tpmes=setmanual&queue=%s&moduser=HammerCloud&comment=HC.Blacklist.set.manual'" % site
self.add_log('> ' + cmd)
if not self.debug:
o = commands.getoutput(cmd)
self.add_log(o)
cmd = "curl -s -k --cert $X509_USER_PROXY 'https://panda.cern.ch:25943/server/controller/query?tpmes=set%s&queue=%s&moduser=HammerCloud&comment=HC.Blacklist.set.%s'" % (new_status, site, new_status)
self.add_log('> ' + cmd)
if not self.debug:
o = commands.getoutput(cmd)
self.add_log(o)
if new_status == 'test':
try:
option = Site.objects.filter(name=site)[0].getSiteOptions_for_site.filter(option_name='last_exclusion')[0]
option.option_value = now
except:
option = SiteOption()
option.option_name = 'last_exclusion'
option.option_value = now
option.site = Site.objects.get(name=site)
option.save()
Client.PandaSites = Client.getSiteSpecs(SITETYPE)[1]
if not self.debug and Client.PandaSites[site]['status'] != new_status:
self.store_log('Error setting %s to %s' % (site, new_status), 'error')
return False
return True
import commands
import datetime
import re
import time
import unittest
import sys
from hc.core.utils.hc import cernmail
from hc.atlas.models import Site, Result, SiteOption, Template, TemplateSite, Test, TestLog, BlacklistEvent
from pandatools import Client
SITETYPE = 'production'
Client.PandaSites = Client.getSiteSpecs(SITETYPE)[1]
DEBUG = True
class Policy:
"""Base class for blacklisting policies."""
def filter_jobs_by_template(self, jobs, template):
"""Filters by template ID."""
return filter(lambda x: x.test.template.id == template, jobs)
def evaluate(self, jobs, site, blacklist, failed_templates_target, num_jobs_per_template):
"""Generic evaluator for XFromY templates."""
if not jobs:
blacklist.site_has_no_jobs(site)
return False
failed_templates = 0
ids = []
for t in blacklist.templates:
template_jobs = self.filter_jobs_by_template(jobs, t)
import re
import sys
sys.path.append('/opt/panda/lib/python2.3/site-packages/')
from pandatools import Client
_SITEMATCH = re.compile('(\S+)_(\S+)')
(e,sitespec) = Client.getSiteSpecs(siteType='all')
for k,v in sitespec.items():
if v['cloud'] == 'UK':
sitematch = _SITEMATCH.match(v['ddm'])
sitename = sitematch.group(1)
q = '(SELECT id FROM pyf_site WHERE name="%s")' % sitename
print "INSERT INTO pyf_pandasite (name,site_id) VALUES ('%s', %s);" % (v['sitename'], q)
