def get_apel_data(self,
year=datetime.datetime.now().year,
month=datetime.datetime.now().month):
srchUrl = 'ApelUrl'
modName = 'get_apel_data'
print "%s: srchUrl: %s" % (modName, srchUrl)
apel_data = []
fed_rpt = []
report_time = None
try:
apel_url = GratiaURLS().GetUrl(srchUrl, year, month)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
else:
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, apel_url)
try:
usock = urllib2.urlopen(apel_url)
except urllib2.HTTPError, e:
print "%s: ======================================" % modName
print "%s: HTTPError: URL server couldn\'t fulfill the URL request." % modName
print modName, ': HTTPError: Error code: ', e.code
print modName, ': HTTPError: ', e.read()
print "%s: ======================================" % modName
raise
except urllib2.URLError, e:
print "%s: ======================================" % modName
print "%s: URLError: Failed to reach the URL server."
print modName, ": URLError: Reason: ", e.reason
print "%s: ======================================" % modName
raise
def getcpuinfodictionary(self):
srchUrl = 'CpuInfoUrl'
modName = 'getcpuinfodictionary'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
urlcpuinfo = GratiaURLS().GetUrl(srchUrl)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
else:
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, urlcpuinfo)
try:
xmldoc = urllib2.urlopen(urlcpuinfo)
except urllib2.HTTPError, e:
print "%s: ======================================" % modName
print "%s: HTTPError: URL server couldn\'t fulfill the URL request." % modName
print modName, ': HTTPError: Error code: ', e.code
print modName, ': HTTPError: ', e.read()
print "%s: ======================================" % modName
raise
except urllib2.URLError, e:
print "%s: ======================================" % modName
print "%s: URLError: Failed to reach the URL server."
print modName, ": URLError: Reason: ", e.reason
print "%s: ======================================" % modName
raise
def __init__(self):
srchUrl = 'OimWLCGSitesUrl'
modName = 'OimWLCGSites'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
self.url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
print "%s: retUrl: %s" % (modName, self.url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
pass
super(OimWLCGSites, self).__init__(self.url)
def gip_validation(self, site):
srchUrl = 'InitGipValidUrl'
modName = 'gip_validation'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
GipValidUrl = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, GipValidUrl)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
doc = urllib2.urlopen(GipValidUrl)
row_re = re.compile("<td valign='middle'>(.*)</td>")
row_re2 = re.compile(site)
info_re = re.compile(
"<td height='30' bgcolor='(.*?)'><a href='(.*?)'>")
in_row = False
result = "Unknown"
link = "#"
info = []
my_facs = []
for line in doc.readlines():
m = row_re.search(line)
if m:
m2 = row_re2.search(m.groups()[0])
if m2:
fac = m.groups()[0]
in_row = True
continue
else:
in_row = False
continue
if in_row:
m = info_re.search(line)
if m:
color, link = m.groups()
if color == 'green':
result = "PASS"
elif color == 'red':
result = "FAIL"
elif color == "black":
result = "Not Reporting"
else:
result = "Unknown"
if fac not in my_facs:
my_facs.append(fac)
info.append((result, GipValidUrl + link, fac))
return info
def __init__(self):
srchUrl = 'OimVoFilterUrl'
modName = 'OimVoParentFilter'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
self.url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, self.url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
super(OimVoParentFilter, self).__init__(self.url)
def __init__(self):
print "================================="
srchUrl = 'VoOwnershipUrl'
modName = 'OimVOOwnership'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
self.url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, self.url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
super(OimVOOwnership, self).__init__(self.url)
def d0(self, weekly=False):
cherrypy.response.headers['Content-Type'] = 'image/png'
srchUrl = 'InitD0Url'
modName = 'd0'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
D0Url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, D0Url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
url = self.metadata.get('d0_csv', D0Url)
url_fp = urllib2.urlopen(url)
metadata = {'croptime':True, 'span':86400, 'pivot_name': 'OSG Site', \
'grouping_name': 'Date', 'column_names': 'Merged Events', \
'title': 'D0 OSG Production'}
if weekly:
metadata['span'] *= 7
fp = cStringIO.StringIO()
data = {}
for line in url_fp.readlines():
info = line.strip().split(',')
grouping_name = time.strptime(info[0], '%y%m%d')
group = datetime.datetime(*grouping_name[:6])
# Change the date to the first day of that week; groups things
# by week instead of the default by day.
if weekly:
weekday = group.isoweekday()
group -= datetime.timedelta(weekday, 0)
print info[0], group
pivot = info[1]
results = int(info[3])
if pivot not in data:
data[pivot] = {}
grouping_dict = data[pivot]
if group not in grouping_dict:
grouping_dict[group] = results
else:
grouping_dict[group] += results
BSB = BasicStackedBar()
BSB(data, fp, metadata)
return fp.getvalue()
def d0_basic(self, weekly=False):
cherrypy.response.headers['Content-Type'] = 'image/png'
srchUrl = 'InitD0Url'
modName = 'd0'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
D0Url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, D0Url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
url_fp = urllib2.urlopen(D0Url)
metadata = {'croptime':False, 'span':86400, 'pivot_name': 'Date', \
'column_names': 'Merged Events', \
'title': 'D0 OSG Production'}
if weekly:
metadata['span'] *= 7
fp = cStringIO.StringIO()
data = {}
for line in url_fp.readlines():
info = line.strip().split(',')
grouping_name = time.strptime(info[0], '%y%m%d')
group = datetime.datetime(*grouping_name[:6])
# Change the date to the first day of that week; groups things
# by week instead of the default by day.
if weekly:
weekday = group.isoweekday()
group -= datetime.timedelta(weekday, 0)
group = to_timestamp(group)
results = int(info[3])
value = data.setdefault(group, 0)
data[group] = value + results
metadata['starttime'] = min(data.values())
time_max = max(data.values())
if weekly:
metadata['endtime'] = time_max + 7 * 86400
else:
metadata['endtime'] = time_max + 1 * 86400
GB = GratiaBar()
GB(data, fp, metadata)
return fp.getvalue()
curs.execute(insert_sql_str, (timestamp, site, status))
conn.commit()
def test_time_list(conn, site):
curs = conn.cursor()
rows = curs.execute(list_tests_str, (site, )).fetchall()
time_tuples = [time.strptime(i[0], '%Y-%m-%d %H:%M:%S') for i in rows]
return [datetime.datetime(*i[:6]) for i in time_tuples]
srchUrl = 'GridScanUrl'
modName = 'gridscan_download'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
gridscan_url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, gridscan_url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
pass
def site_listing():
query = urllib.urlencode({"get": "set1"})
doc = urllib2.urlopen(gridscan_url + query)
retval = {}
for line in doc:
id, site, ce = line.split(',')
retval[site] = id
return retval
import time
import urllib
import urllib2
import datetime
import os.path
import string
from gratia.web.gratia_urls import GratiaURLS
from xml.dom.minidom import parse
print "===================================="
srchUrl = 'DashboardUrl'
modName = 'atlas_statistics'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
dashboard_url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, dashboard_url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
pass
print "===================================="
dashboard_t0_url = dashboard_url
def parse_and_print(url, to='.*'):
to = re.compile(to)
req = urllib2.Request(url, headers={"Accept": "text/xml"})
dom = parse(urllib2.urlopen(req))
total = 0
import urllib
import urllib2
import datetime
import os.path
import string
from gratia.web.gratia_urls import GratiaURLS
from xml.dom.minidom import parse
from atlas_statistics import dostats as do_atlas_stats
srchUrl = 'VoHoursUrl'
modName = 'metric_thumbnails'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
vo_hours = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, vo_hours)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
pass
srchUrl = 'VoCountUrl'
modName = 'gridscan_download'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
vo_count = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, vo_count)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
import urllib2
import datetime
import os.path
import string
from gratia.web.gratia_urls import GratiaURLS
from xml.dom.minidom import parse
from graphtool.tools.common import convert_to_datetime, expand_string
from graphtool.graphs.common_graphs import ScatterPlot
srchUrl = 'AvailSummaryUrl'
modName = 'avail_vs_work'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
avail_url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, avail_url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
pass
srchUrl = 'FacilityWorkUrl'
modName = 'avail_vs_work'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
work_url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, work_url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
def wlcg_pledges(self, month=None, year=None):
thisyear = str(year)
debug=1
sites_per_accountname={}
fednames_and_accounts={}
cms_fed = {}
atlas_fed = {}
alice_fed = {}
cms_pledge = {}
atlas_pledge = {}
alice_pledge = {}
srchUrl = 'TopologyUrl'
modName = 'wlcg_pledges'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
url = GratiaURLS().GetUrl(srchUrl, year, month)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
print "%s: retUrl: %s" % (modName, url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
pass
response = urllib2.urlopen(url)
s = response.read()
x = simplejson.loads(s)
for obj in x:
if('USA'==obj['Country']):
try:
sites_per_accountname[obj['FederationAccountingName']].append(obj['Site'])
except KeyError:
sites_per_accountname[obj['FederationAccountingName']]=[obj['Site']]
fednames_and_accounts[obj['Federation']]=obj['FederationAccountingName']#Name is key
if('USA'==obj['Country'] and obj['Federation'].find('CMS') >= 0):
cms_fed[obj['FederationAccountingName']]=1;
if('USA'==obj['Country'] and obj['Federation'].find('ATLAS') >= 0):
atlas_fed[obj['FederationAccountingName']]=1;
if('USA'==obj['Country'] and obj['Federation'].find('ALICE') >= 0):
alice_fed[obj['FederationAccountingName']]=1;
if(debug > 0):
print "debug: ======================================================"
print "debug: sites_per_accountname: "
print "debug: ======================================================"
for key in sites_per_accountname:
print "\ndebug: FederationName: %s" % key
print "debug: \t\t ResourceGroups:"
for key2 in sites_per_accountname[key]:
print "debug: \t\t\t %s"%(key2)
print "\ndebug: ======================================================"
print "debug: cms_fed: CMS Federation Names"
print "debug: ======================================================"
for key in cms_fed:
print "debug: \t\t %s " % key
print "\ndebug: ======================================================"
print "debug: atlas_fed: Atlas Federation Names"
print "debug: ======================================================"
for key in atlas_fed:
print "debug: \t\t %s " % key
print "\ndebug: ======================================================"
print "debug: alice_fed: Alice Federation Names"
print "debug: ======================================================"
for key in alice_fed:
print "debug: \t\t %s " % key
srchUrl = 'PledgeUrl'
modName = 'wlcg_pledges'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
url = GratiaURLS().GetUrl(srchUrl, year, month)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
print "%s: retUrl: %s" % (modName, url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
pass
response = urllib2.urlopen(url)
s = response.read()
x = simplejson.loads(s)
for obj in x:
if('USA'==obj['Country'] and 'HEP-SPEC06' == obj['PledgeUnit']):
try:
int(obj['ATLAS']) #atlas number exists
atlas_pledge[fednames_and_accounts[obj['Federation']]] = \
{'pledge': obj['ATLAS'], 'ResourceGroups': \
sites_per_accountname[fednames_and_accounts[obj['Federation']]]}
except:
None
try:
int(obj['CMS']) #cms number exists
cms_pledge[fednames_and_accounts[obj['Federation']]] = \
{'pledge': obj['CMS'], 'ResourceGroups': \
sites_per_accountname[fednames_and_accounts[obj['Federation']]]}
except:
None
try:
int(obj['ALICE']) #alice number exists
alice_pledge[fednames_and_accounts[obj['Federation']]] = \
{'pledge': obj['ALICE'], 'ResourceGroups': \
sites_per_accountname[fednames_and_accounts[obj['Federation']]]}
except:
None
if(debug > 0):
print "debug: ================================================================="
print "debug: Final Data Check"
print "debug: ================================================================="
print "debug: cms_pledge: "
print "debug: ======================================================"
for key in cms_pledge:
print "\ndebug: FederationName: %s Pledge: %s"%(key, cms_pledge[key]['pledge'])
print "debug: \t\t Resource Groups:"
for key2 in cms_pledge[key]['ResourceGroups']:
print "debug: \t\t\t %s" % key2
print "debug: ======================================================"
print "debug: atlas_pledge: "
print "debug: ======================================================"
for key in atlas_pledge:
print "\ndebug: FederationName: %s Pledge: %s"%(key, atlas_pledge[key]['pledge'])
print "debug: \t\t Resource Groups:"
for key2 in atlas_pledge[key]['ResourceGroups']:
print "debug: \t\t %s" % key2
print "debug: ======================================================"
print "debug: alice_pledge: "
print "debug: ======================================================"
for key in alice_pledge:
print "\ndebug: FederationName: %s Pledge: %s"%(key, alice_pledge[key]['pledge'])
print "degub: \t\t Resource Groups:"
for key2 in alice_pledge[key]['ResourceGroups']:
print "debug: \t\t\t %s" % key2
if(debug > 0):
print "\ndebug: ======================================================"
print "debug: cms_fed: CMS Federation Accounting Names"
print "debug: ======================================================"
for key in cms_fed:
print "debug: \t\t %s" % key
print "\ndebug: ======================================================"
print "debug: atlas_fed: Atlas Federation Accounting Names"
print "debug: ======================================================"
for key in atlas_fed:
print "debug: \t\t %s " % key
print "\ndebug: ======================================================"
print "debug: alice_fed: Alice Federation Accounting Names"
print "debug: ======================================================"
for key in alice_fed:
print "debug: alice_fed: ",key
return atlas_pledge, cms_pledge, alice_pledge
import string
from gratia.web.gratia_urls import GratiaURLS
from xml.dom.minidom import parse
ok = "OK"
warn = "WARNING"
error = "CRITICAL"
unknown = "UNKNOWN"
expired = "EXPIRED"
srchUrl = 'LastStatusUrl'
modName = 'rsv_nagios'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
last_status_url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, last_status_url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
pass
def simple_results_parser(dom):
data = {}
pivots = dom.getElementsByTagName('pivot')
for pivot_dom in pivots:
pivot = pivot_dom.getAttribute('name')
data_dom = pivot_dom.getElementsByTagName('d')[0]
datum = str(data_dom.firstChild.data)
data[pivot] = datum
import traceback
from sets import Set
from httplib import HTTPSConnection
from xml.dom.minidom import parse
from ZSI.client import Binding
from ZSI import TC
import os.path
import string
from gratia.web.gratia_urls import GratiaURLS
srchUrl = 'GumsTemplateUrl'
modName = 'voms_init'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
gums_template_url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
print "%s: retUrl: %s" % (modName, gums_template_url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
pass
class HTTPSConnection2(HTTPSConnection):
def __init__(self, host, port=443):
if 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ:
certfile = os.environ['X509_USER_CERT']
keyfile = os.environ['X509_USER_KEY']
elif 'X509_USER_PROXY' in os.environ:
keyfile = os.environ['X509_USER_PROXY']
certfile = os.environ['X509_USER_PROXY']
def get_apel_data_jot(self, year=None, month=None):
apel_data = []
report_time = None
if (year is not None) and (month is not None):
srchUrl = 'ApelUrl'
modName = 'get_apel_data_jot'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
apel_url = GratiaURLS().GetUrl(srchUrl, year, month)
except:
print "%s: ======================================" % modName
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: ======================================" % modName
pass
else:
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, apel_url)
try:
usock = urllib2.urlopen(apel_url)
except urllib2.HTTPError, e:
print "%s: ======================================" % modName
print "%s: HTTPError: URL server couldn\'t fulfill the URL request." % modName
print modName, ': HTTPError: Error code: ', e.code
print modName, ': HTTPError: ', e.read()
print "%s: ======================================" % modName
raise
except urllib2.URLError, e:
print "%s: ======================================" % modName
print "%s: URLError: Failed to reach the URL server."
print modName, ": URLError: Reason: ", e.reason
print "%s: ======================================" % modName
raise
else:
print "%s: ======================================" % modName
print "%s: Successful URL open to: %s" % (modName,
apel_url)
print "%s: ======================================" % modName
data = usock.read()
usock.close()
datafields = []
numcells = 13
for i in range(numcells):
datafields.append(0)
datafields[0] = "ResourceGroup"
datafields[1] = "NormFactor"
datafields[2] = "LCGUserVO"
datafields[3] = "Njobs"
datafields[4] = "SumCPU"
datafields[5] = "SumWCT"
datafields[6] = "Norm_CPU"
datafields[7] = "Norm_WCT"
datafields[8] = "RecordStart"
datafields[9] = "RecordEnd"
datafields[10] = "MeasurementDate"
datafields[11] = "FederationName"
datafields[12] = "ResourcesReporting"
linesrec = data.split('\n')
for line in linesrec:
thisTuple = line.split('\t')
print "thisTuple: %s" % thisTuple
count = 0
info = {}
for thisField in thisTuple:
print "thisField: %s" % thisField
if (thisField.strip() == ""):
continue
if (count < numcells):
info[datafields[count]] = thisField
print "info[datafields[%d]]= %s" % (count,
thisField)
if count < numcells and datafields[
count] == 'MeasurementDate' and report_time == None:
report_time = thisField
count = count + 1
if (not info):
continue
info['month'] = month
info['year'] = year
apel_data.append(info)
print "info: %s" % info
