def get_apel_data(self, year=datetime.datetime.now().year, month=datetime.datetime.now().month): srchUrl = 'ApelUrl' modName = 'get_apel_data' print "%s: srchUrl: %s" % (modName, srchUrl) apel_data = [] fed_rpt = [] report_time = None try: apel_url = GratiaURLS().GetUrl(srchUrl, year, month) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass else: print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, apel_url) try: usock = urllib2.urlopen(apel_url) except urllib2.HTTPError, e: print "%s: ======================================" % modName print "%s: HTTPError: URL server couldn\'t fulfill the URL request." % modName print modName, ': HTTPError: Error code: ', e.code print modName, ': HTTPError: ', e.read() print "%s: ======================================" % modName raise except urllib2.URLError, e: print "%s: ======================================" % modName print "%s: URLError: Failed to reach the URL server." print modName, ": URLError: Reason: ", e.reason print "%s: ======================================" % modName raise
def getcpuinfodictionary(self): srchUrl = 'CpuInfoUrl' modName = 'getcpuinfodictionary' print "%s: srchUrl: %s" % (modName, srchUrl) try: urlcpuinfo = GratiaURLS().GetUrl(srchUrl) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass else: print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, urlcpuinfo) try: xmldoc = urllib2.urlopen(urlcpuinfo) except urllib2.HTTPError, e: print "%s: ======================================" % modName print "%s: HTTPError: URL server couldn\'t fulfill the URL request." % modName print modName, ': HTTPError: Error code: ', e.code print modName, ': HTTPError: ', e.read() print "%s: ======================================" % modName raise except urllib2.URLError, e: print "%s: ======================================" % modName print "%s: URLError: Failed to reach the URL server." print modName, ": URLError: Reason: ", e.reason print "%s: ======================================" % modName raise
def __init__(self): srchUrl = 'OimWLCGSitesUrl' modName = 'OimWLCGSites' print "%s: srchUrl: %s" % (modName, srchUrl) try: self.url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl) print "%s: retUrl: %s" % (modName, self.url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl) pass super(OimWLCGSites, self).__init__(self.url)
def gip_validation(self, site): srchUrl = 'InitGipValidUrl' modName = 'gip_validation' print "%s: srchUrl: %s" % (modName, srchUrl) try: GipValidUrl = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, GipValidUrl) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass doc = urllib2.urlopen(GipValidUrl) row_re = re.compile("<td valign='middle'>(.*)</td>") row_re2 = re.compile(site) info_re = re.compile( "<td height='30' bgcolor='(.*?)'><a href='(.*?)'>") in_row = False result = "Unknown" link = "#" info = [] my_facs = [] for line in doc.readlines(): m = row_re.search(line) if m: m2 = row_re2.search(m.groups()[0]) if m2: fac = m.groups()[0] in_row = True continue else: in_row = False continue if in_row: m = info_re.search(line) if m: color, link = m.groups() if color == 'green': result = "PASS" elif color == 'red': result = "FAIL" elif color == "black": result = "Not Reporting" else: result = "Unknown" if fac not in my_facs: my_facs.append(fac) info.append((result, GipValidUrl + link, fac)) return info
def __init__(self): srchUrl = 'OimVoFilterUrl' modName = 'OimVoParentFilter' print "%s: srchUrl: %s" % (modName, srchUrl) try: self.url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, self.url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass super(OimVoParentFilter, self).__init__(self.url)
def __init__(self): print "=================================" srchUrl = 'VoOwnershipUrl' modName = 'OimVOOwnership' print "%s: srchUrl: %s" % (modName, srchUrl) try: self.url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, self.url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass super(OimVOOwnership, self).__init__(self.url)
def d0(self, weekly=False): cherrypy.response.headers['Content-Type'] = 'image/png' srchUrl = 'InitD0Url' modName = 'd0' print "%s: srchUrl: %s" % (modName, srchUrl) try: D0Url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, D0Url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass url = self.metadata.get('d0_csv', D0Url) url_fp = urllib2.urlopen(url) metadata = {'croptime':True, 'span':86400, 'pivot_name': 'OSG Site', \ 'grouping_name': 'Date', 'column_names': 'Merged Events', \ 'title': 'D0 OSG Production'} if weekly: metadata['span'] *= 7 fp = cStringIO.StringIO() data = {} for line in url_fp.readlines(): info = line.strip().split(',') grouping_name = time.strptime(info[0], '%y%m%d') group = datetime.datetime(*grouping_name[:6]) # Change the date to the first day of that week; groups things # by week instead of the default by day. if weekly: weekday = group.isoweekday() group -= datetime.timedelta(weekday, 0) print info[0], group pivot = info[1] results = int(info[3]) if pivot not in data: data[pivot] = {} grouping_dict = data[pivot] if group not in grouping_dict: grouping_dict[group] = results else: grouping_dict[group] += results BSB = BasicStackedBar() BSB(data, fp, metadata) return fp.getvalue()
def d0_basic(self, weekly=False): cherrypy.response.headers['Content-Type'] = 'image/png' srchUrl = 'InitD0Url' modName = 'd0' print "%s: srchUrl: %s" % (modName, srchUrl) try: D0Url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, D0Url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass url_fp = urllib2.urlopen(D0Url) metadata = {'croptime':False, 'span':86400, 'pivot_name': 'Date', \ 'column_names': 'Merged Events', \ 'title': 'D0 OSG Production'} if weekly: metadata['span'] *= 7 fp = cStringIO.StringIO() data = {} for line in url_fp.readlines(): info = line.strip().split(',') grouping_name = time.strptime(info[0], '%y%m%d') group = datetime.datetime(*grouping_name[:6]) # Change the date to the first day of that week; groups things # by week instead of the default by day. if weekly: weekday = group.isoweekday() group -= datetime.timedelta(weekday, 0) group = to_timestamp(group) results = int(info[3]) value = data.setdefault(group, 0) data[group] = value + results metadata['starttime'] = min(data.values()) time_max = max(data.values()) if weekly: metadata['endtime'] = time_max + 7 * 86400 else: metadata['endtime'] = time_max + 1 * 86400 GB = GratiaBar() GB(data, fp, metadata) return fp.getvalue()
curs.execute(insert_sql_str, (timestamp, site, status)) conn.commit() def test_time_list(conn, site): curs = conn.cursor() rows = curs.execute(list_tests_str, (site, )).fetchall() time_tuples = [time.strptime(i[0], '%Y-%m-%d %H:%M:%S') for i in rows] return [datetime.datetime(*i[:6]) for i in time_tuples] srchUrl = 'GridScanUrl' modName = 'gridscan_download' print "%s: srchUrl: %s" % (modName, srchUrl) try: gridscan_url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, gridscan_url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass def site_listing(): query = urllib.urlencode({"get": "set1"}) doc = urllib2.urlopen(gridscan_url + query) retval = {} for line in doc: id, site, ce = line.split(',') retval[site] = id return retval
import time import urllib import urllib2 import datetime import os.path import string from gratia.web.gratia_urls import GratiaURLS from xml.dom.minidom import parse print "====================================" srchUrl = 'DashboardUrl' modName = 'atlas_statistics' print "%s: srchUrl: %s" % (modName, srchUrl) try: dashboard_url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, dashboard_url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass print "====================================" dashboard_t0_url = dashboard_url def parse_and_print(url, to='.*'): to = re.compile(to) req = urllib2.Request(url, headers={"Accept": "text/xml"}) dom = parse(urllib2.urlopen(req)) total = 0
import urllib import urllib2 import datetime import os.path import string from gratia.web.gratia_urls import GratiaURLS from xml.dom.minidom import parse from atlas_statistics import dostats as do_atlas_stats srchUrl = 'VoHoursUrl' modName = 'metric_thumbnails' print "%s: srchUrl: %s" % (modName, srchUrl) try: vo_hours = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, vo_hours) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass srchUrl = 'VoCountUrl' modName = 'gridscan_download' print "%s: srchUrl: %s" % (modName, srchUrl) try: vo_count = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, vo_count) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
import urllib2 import datetime import os.path import string from gratia.web.gratia_urls import GratiaURLS from xml.dom.minidom import parse from graphtool.tools.common import convert_to_datetime, expand_string from graphtool.graphs.common_graphs import ScatterPlot srchUrl = 'AvailSummaryUrl' modName = 'avail_vs_work' print "%s: srchUrl: %s" % (modName, srchUrl) try: avail_url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, avail_url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass srchUrl = 'FacilityWorkUrl' modName = 'avail_vs_work' print "%s: srchUrl: %s" % (modName, srchUrl) try: work_url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, work_url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl)
def wlcg_pledges(self, month=None, year=None): thisyear = str(year) debug=1 sites_per_accountname={} fednames_and_accounts={} cms_fed = {} atlas_fed = {} alice_fed = {} cms_pledge = {} atlas_pledge = {} alice_pledge = {} srchUrl = 'TopologyUrl' modName = 'wlcg_pledges' print "%s: srchUrl: %s" % (modName, srchUrl) try: url = GratiaURLS().GetUrl(srchUrl, year, month) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl) print "%s: retUrl: %s" % (modName, url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl) pass response = urllib2.urlopen(url) s = response.read() x = simplejson.loads(s) for obj in x: if('USA'==obj['Country']): try: sites_per_accountname[obj['FederationAccountingName']].append(obj['Site']) except KeyError: sites_per_accountname[obj['FederationAccountingName']]=[obj['Site']] fednames_and_accounts[obj['Federation']]=obj['FederationAccountingName']#Name is key if('USA'==obj['Country'] and obj['Federation'].find('CMS') >= 0): cms_fed[obj['FederationAccountingName']]=1; if('USA'==obj['Country'] and obj['Federation'].find('ATLAS') >= 0): atlas_fed[obj['FederationAccountingName']]=1; if('USA'==obj['Country'] and obj['Federation'].find('ALICE') >= 0): alice_fed[obj['FederationAccountingName']]=1; if(debug > 0): print "debug: ======================================================" print "debug: sites_per_accountname: " print "debug: ======================================================" for key in sites_per_accountname: print "\ndebug: FederationName: %s" % key print "debug: \t\t ResourceGroups:" for key2 in sites_per_accountname[key]: print "debug: \t\t\t %s"%(key2) print "\ndebug: ======================================================" print "debug: cms_fed: CMS Federation Names" print "debug: ======================================================" for key in cms_fed: print "debug: \t\t %s " % key print "\ndebug: ======================================================" print "debug: atlas_fed: Atlas Federation Names" print "debug: ======================================================" for key in atlas_fed: print "debug: \t\t %s " % key print "\ndebug: ======================================================" print "debug: alice_fed: Alice Federation Names" print "debug: ======================================================" for key in alice_fed: print "debug: \t\t %s " % key srchUrl = 'PledgeUrl' modName = 'wlcg_pledges' print "%s: srchUrl: %s" % (modName, srchUrl) try: url = GratiaURLS().GetUrl(srchUrl, year, month) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl) print "%s: retUrl: %s" % (modName, url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl) pass response = urllib2.urlopen(url) s = response.read() x = simplejson.loads(s) for obj in x: if('USA'==obj['Country'] and 'HEP-SPEC06' == obj['PledgeUnit']): try: int(obj['ATLAS']) #atlas number exists atlas_pledge[fednames_and_accounts[obj['Federation']]] = \ {'pledge': obj['ATLAS'], 'ResourceGroups': \ sites_per_accountname[fednames_and_accounts[obj['Federation']]]} except: None try: int(obj['CMS']) #cms number exists cms_pledge[fednames_and_accounts[obj['Federation']]] = \ {'pledge': obj['CMS'], 'ResourceGroups': \ sites_per_accountname[fednames_and_accounts[obj['Federation']]]} except: None try: int(obj['ALICE']) #alice number exists alice_pledge[fednames_and_accounts[obj['Federation']]] = \ {'pledge': obj['ALICE'], 'ResourceGroups': \ sites_per_accountname[fednames_and_accounts[obj['Federation']]]} except: None if(debug > 0): print "debug: =================================================================" print "debug: Final Data Check" print "debug: =================================================================" print "debug: cms_pledge: " print "debug: ======================================================" for key in cms_pledge: print "\ndebug: FederationName: %s Pledge: %s"%(key, cms_pledge[key]['pledge']) print "debug: \t\t Resource Groups:" for key2 in cms_pledge[key]['ResourceGroups']: print "debug: \t\t\t %s" % key2 print "debug: ======================================================" print "debug: atlas_pledge: " print "debug: ======================================================" for key in atlas_pledge: print "\ndebug: FederationName: %s Pledge: %s"%(key, atlas_pledge[key]['pledge']) print "debug: \t\t Resource Groups:" for key2 in atlas_pledge[key]['ResourceGroups']: print "debug: \t\t %s" % key2 print "debug: ======================================================" print "debug: alice_pledge: " print "debug: ======================================================" for key in alice_pledge: print "\ndebug: FederationName: %s Pledge: %s"%(key, alice_pledge[key]['pledge']) print "degub: \t\t Resource Groups:" for key2 in alice_pledge[key]['ResourceGroups']: print "debug: \t\t\t %s" % key2 if(debug > 0): print "\ndebug: ======================================================" print "debug: cms_fed: CMS Federation Accounting Names" print "debug: ======================================================" for key in cms_fed: print "debug: \t\t %s" % key print "\ndebug: ======================================================" print "debug: atlas_fed: Atlas Federation Accounting Names" print "debug: ======================================================" for key in atlas_fed: print "debug: \t\t %s " % key print "\ndebug: ======================================================" print "debug: alice_fed: Alice Federation Accounting Names" print "debug: ======================================================" for key in alice_fed: print "debug: alice_fed: ",key return atlas_pledge, cms_pledge, alice_pledge
import string from gratia.web.gratia_urls import GratiaURLS from xml.dom.minidom import parse ok = "OK" warn = "WARNING" error = "CRITICAL" unknown = "UNKNOWN" expired = "EXPIRED" srchUrl = 'LastStatusUrl' modName = 'rsv_nagios' print "%s: srchUrl: %s" % (modName, srchUrl) try: last_status_url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, last_status_url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass def simple_results_parser(dom): data = {} pivots = dom.getElementsByTagName('pivot') for pivot_dom in pivots: pivot = pivot_dom.getAttribute('name') data_dom = pivot_dom.getElementsByTagName('d')[0] datum = str(data_dom.firstChild.data) data[pivot] = datum
import traceback from sets import Set from httplib import HTTPSConnection from xml.dom.minidom import parse from ZSI.client import Binding from ZSI import TC import os.path import string from gratia.web.gratia_urls import GratiaURLS srchUrl = 'GumsTemplateUrl' modName = 'voms_init' print "%s: srchUrl: %s" % (modName, srchUrl) try: gums_template_url = GratiaURLS().GetUrl(srchUrl) print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, gums_template_url) except: print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) pass class HTTPSConnection2(HTTPSConnection): def __init__(self, host, port=443): if 'X509_USER_CERT' in os.environ and 'X509_USER_KEY' in os.environ: certfile = os.environ['X509_USER_CERT'] keyfile = os.environ['X509_USER_KEY'] elif 'X509_USER_PROXY' in os.environ: keyfile = os.environ['X509_USER_PROXY'] certfile = os.environ['X509_USER_PROXY']
def get_apel_data_jot(self, year=None, month=None): apel_data = [] report_time = None if (year is not None) and (month is not None): srchUrl = 'ApelUrl' modName = 'get_apel_data_jot' print "%s: srchUrl: %s" % (modName, srchUrl) try: apel_url = GratiaURLS().GetUrl(srchUrl, year, month) except: print "%s: ======================================" % modName print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: ======================================" % modName pass else: print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName, srchUrl) print "%s: retUrl: %s" % (modName, apel_url) try: usock = urllib2.urlopen(apel_url) except urllib2.HTTPError, e: print "%s: ======================================" % modName print "%s: HTTPError: URL server couldn\'t fulfill the URL request." % modName print modName, ': HTTPError: Error code: ', e.code print modName, ': HTTPError: ', e.read() print "%s: ======================================" % modName raise except urllib2.URLError, e: print "%s: ======================================" % modName print "%s: URLError: Failed to reach the URL server." print modName, ": URLError: Reason: ", e.reason print "%s: ======================================" % modName raise else: print "%s: ======================================" % modName print "%s: Successful URL open to: %s" % (modName, apel_url) print "%s: ======================================" % modName data = usock.read() usock.close() datafields = [] numcells = 13 for i in range(numcells): datafields.append(0) datafields[0] = "ResourceGroup" datafields[1] = "NormFactor" datafields[2] = "LCGUserVO" datafields[3] = "Njobs" datafields[4] = "SumCPU" datafields[5] = "SumWCT" datafields[6] = "Norm_CPU" datafields[7] = "Norm_WCT" datafields[8] = "RecordStart" datafields[9] = "RecordEnd" datafields[10] = "MeasurementDate" datafields[11] = "FederationName" datafields[12] = "ResourcesReporting" linesrec = data.split('\n') for line in linesrec: thisTuple = line.split('\t') print "thisTuple: %s" % thisTuple count = 0 info = {} for thisField in thisTuple: print "thisField: %s" % thisField if (thisField.strip() == ""): continue if (count < numcells): info[datafields[count]] = thisField print "info[datafields[%d]]= %s" % (count, thisField) if count < numcells and datafields[ count] == 'MeasurementDate' and report_time == None: report_time = thisField count = count + 1 if (not info): continue info['month'] = month info['year'] = year apel_data.append(info) print "info: %s" % info