def __init__(self, app, baseurl, mount, cacertdir = "/etc/grid-security/certificates", minreq = 1000, interval = 300, instance = "test"):
Thread.__init__(self, name = "LdapSync")
self.sectoken = "".join(random.sample(string.letters, 30))
self._inturl = "http://localhost:%d%s/%s/ldapsync" % \
(app.srvconfig.port, mount, instance)
self._headers = \
fake_authz_headers(open(app.srvconfig.tools.cms_auth.key_file).read(),
method = "Internal", login = self.sectoken,
name = self.__class__.__name__, dn = None,
roles = {"Global Admin": {"group": ["global"]}}) \
+ [("Accept", "application/json")]
self._cv = Condition()
if isinstance(baseurl, str):
self._baseurl = baseurl
self._cacertdir = cacertdir
self._minreq = minreq
self._interval = interval
self._stopme = False
self._full = (0, [], [])
self._warnings = {}
self._intreq = RequestManager(num_connections = 2,
user_agent = self._ident,
handle_init = self._handle_init,
request_respond = self._respond,
request_init = self._int_init)
cherrypy.engine.subscribe("stop", self.stop)
cherrypy.engine.subscribe("start", self.start)
def __init__(self, app, baseurl, mount, cacertdir = "/etc/grid-security/certificates", minreq = 1000, interval = 300, instance = "test"):
Thread.__init__(self, name = "LdapSync")
self.sectoken = "".join(random.sample(string.letters, 30))
self._inturl = "http://localhost:%d%s/%s/ldapsync" % \
(app.srvconfig.port, mount, instance)
self._headers = \
fake_authz_headers(open(app.srvconfig.tools.cms_auth.key_file).read(),
method = "Internal", login = self.sectoken,
name = self.__class__.__name__, dn = None,
roles = {"Global Admin": {"group": ["global"]}}) \
+ [("Accept", "application/json")]
self._cv = Condition()
if isinstance(baseurl, str):
self._baseurl = baseurl
self._cacertdir = cacertdir
self._minreq = minreq
self._interval = interval
self._stopme = False
self._full = (0, [], [])
self._warnings = {}
self._intreq = RequestManager(num_connections = 2,
user_agent = self._ident,
handle_init = self._handle_init,
request_respond = self._respond,
request_init = self._int_init)
cherrypy.engine.subscribe("stop", self.stop)
cherrypy.engine.subscribe("start", self.start)
class RebusFetchThread(Thread):
"""A task thread to synchronise federation pledges from REBUS. This runs on
a single node only in the cluster."""
_baseurl = "http://gstat-wlcg.cern.ch/apps/pledges/resources/"
_ident = "SiteDB/%s Python/%s" % \
(os.environ["SITEDB_VERSION"], ".".join(map(str, sys.version_info[:3])))
def __init__(self,
app,
baseurl,
mount,
cacertdir="/etc/grid-security/certificates",
minreq=1000,
interval=300,
instance="test"):
Thread.__init__(self, name="RebusFetch")
self.sectoken = "".join(random.sample(string.letters, 30))
self._inturl = "http://localhost:%d%s/%s/rebusfetch" % \
(app.srvconfig.port, mount, instance)
self._headers = \
fake_authz_headers(open(app.srvconfig.tools.cms_auth.key_file).read(),
method = "Internal", login = self.sectoken,
name = self.__class__.__name__, dn = None,
roles = {"Global Admin": {"group": ["global"]}}) \
+ [("Accept", "application/json")]
self._cv = Condition()
if isinstance(baseurl, str):
self._baseurl = baseurl
self._cacertdir = cacertdir
self._minreq = minreq
self._interval = interval
self._stopme = False
self._full = (0, [], [])
self._warnings = {}
self._intreq = RequestManager(num_connections=2,
user_agent=self._ident,
handle_init=self._handle_init,
request_respond=self._respond,
request_init=self._int_init)
cherrypy.engine.subscribe("stop", self.stop)
cherrypy.engine.subscribe("start", self.start)
def status(self):
"""Get the processing status. Returns time of last successful full
synchronisation with REBUS."""
with self._cv:
return self._full[:]
def stop(self, *args):
"""Tell the task thread to quit."""
with self._cv:
self._stopme = True
self._cv.notifyAll()
def run(self):
"""Run synchronisation thread."""
while True:
now = time.time()
until = now + self._interval
try:
self._sync(now)
except Exception as e:
cherrypy.log("SYNC ERROR %s.%s REBUS sync failed %s" %
(getattr(e, "__module__", "__builtins__"),
e.__class__.__name__, str(e)))
for line in format_exc().rstrip().split("\n"):
cherrypy.log(" " + line)
with self._cv:
while not self._stopme and now < until:
self._cv.wait(until - now)
now = time.time()
if self._stopme:
return
def _validate(self, input, type, regexp, now):
"""Convenience method to validate ldap data"""
if isinstance(input, type):
m = regexp.match(input)
if m: return m
if input not in self._warnings:
cherrypy.log("WARNING: REBUS data failed validation: '%s'" % input)
self._warnings[input] = now
return None
def _sync(self, now):
"""Perform full synchronisation."""
# Delete warnings older than 24 hours
for k, v in self._warnings.items():
if v < now - 86400:
del self._warnings[k]
result = []
# Get the user information from CERN/LDAP
ldresult = self._ldget(self._baseurl)
# get data from oracle database
# Process each user record returned
rows = []
id = 0
for name, values in ldresult.iteritems():
for year, val1 in values["pledges"].iteritems():
i = {
'name': name,
'country': values["country"],
'year': str(year),
'cpu': str(0),
'disk': str(0),
'tape': str(0)
}
if 'CPU' in val1.keys():
i['cpu'] = str(val1['CPU'] / float(1000))
if 'Disk' in val1.keys():
i['disk'] = str(val1['Disk'])
if 'Tape' in val1.keys():
i['tape'] = str(val1['Tape'])
rows.append(i)
gett = self._intreq.put(("PUT", rows, result))
gettt = self._intreq.process()
def _ldget(self, url):
"""Get data from REBUS."""
year_next = date.today().year + 2
result = self._read_rebus_data(2008, year_next)
return result
def _handle_init(self, c):
"""Initialise curl handle `c`."""
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
def _respond(self, c):
"""Respond to data on curl handle `c`."""
code = c.getinfo(pycurl.HTTP_CODE)
if code != 200:
raise RuntimeError("HTTP status %d for %s" %
(code, c.getinfo(pycurl.EFFECTIVE_URL)))
c.result.append(c.buffer.getvalue())
def _int_init(self, c, method, rows, result):
"""Initialise curl handle `c` for an internal REST API request."""
if method == "PUT":
type, body = self._encode(rows)
headers = self._headers[:] + [("Content-Type", type),
("Content-Length", str(len(body)))]
c.setopt(pycurl.POST, 0)
c.setopt(pycurl.UPLOAD, 1)
c.setopt(pycurl.URL, self._inturl)
c.setopt(pycurl.HTTPHEADER, ["%s: %s" % h for h in headers])
c.setopt(pycurl.READFUNCTION, StringIO(body).read)
c.result = result
elif method == "GET":
headers = self._headers[:]
c.setopt(pycurl.URL, self._inturl)
c.setopt(pycurl.HTTPHEADER, ["%s: %s" % h for h in headers])
c.result = result
else:
assert False, "Unsupported method"
def _encode(self, rows):
"""Encode dictionaries in `rows` for POST/PUT body as a HTML form."""
body, sep = "", ""
for obj in rows:
for key, value in obj.iteritems():
body += "%s%s=%s" % (sep, key,
urllib2.quote(value.encode("utf-8")))
sep = "&"
return ("application/x-www-form-urlencoded", body)
def _read_rebus_data(self, year, yearTo):
"""REBUS json data fetch from 2008 to year.now + 2. All data returned in dictionary."""
data = {}
for x in range(year, yearTo):
url = "http://wlcg-rebus.cern.ch/apps/pledges/resources/" + str(
x) + "/all/json"
req = urllib2.Request(url)
opener = urllib2.build_opener()
f = opener.open(req)
pledges = json.loads(f.read())
for index, item in enumerate(pledges):
federname = item["Federation"]
cms = item["CMS"]
pledgetype = item["PledgeType"]
country = item["Country"]
if cms and pledgetype:
if federname in data.keys():
if x in data[federname]["pledges"].keys():
data[federname]["pledges"][x][pledgetype] = cms
else:
data[federname]["pledges"][x] = {
pledgetype: cms
}
else:
data[federname] = {
"country": country,
"pledges": {}
}
data[federname]["pledges"] = {
x: {
pledgetype: cms
}
}
return data
class LdapSyncThread(Thread):
"""A task thread to synchronise SiteDB from CERN/LDAP. This runs on
a single node only in the cluster."""
_baseurl = "ldaps://xldap.cern.ch:636"
_ident = "SiteDB/%s Python/%s" % \
(os.environ["SITEDB_VERSION"], ".".join(map(str, sys.version_info[:3])))
# The buggy ca.cern.ch user interface allows to put anything, so some users
# have uploaded CA certificates or even SSH keys. So try to ignore them.
RX_ALTDN = re.compile(
r"(?iu)^X509:.*<S>(([A-Z]+=([-\w _@'.()/]+),?)*(?<!berosservice|CERN Root CA|on Authority|s,CN=lsfcert|s,CN=acronmc))$"
)
def __init__(self,
app,
baseurl,
mount,
cacertdir="/etc/grid-security/certificates",
minreq=1000,
interval=300,
instance="test"):
Thread.__init__(self, name="LdapSync")
self.sectoken = "".join(random.sample(string.letters, 30))
self._inturl = "http://localhost:%d%s/%s/ldapsync" % \
(app.srvconfig.port, mount, instance)
self._headers = \
fake_authz_headers(open(app.srvconfig.tools.cms_auth.key_file).read(),
method = "Internal", login = self.sectoken,
name = self.__class__.__name__, dn = None,
roles = {"Global Admin": {"group": ["global"]}}) \
+ [("Accept", "application/json")]
self._cv = Condition()
if isinstance(baseurl, str):
self._baseurl = baseurl
self._cacertdir = cacertdir
self._minreq = minreq
self._interval = interval
self._stopme = False
self._full = (0, [], [])
self._warnings = {}
self._intreq = RequestManager(num_connections=2,
user_agent=self._ident,
handle_init=self._handle_init,
request_respond=self._respond,
request_init=self._int_init)
cherrypy.engine.subscribe("stop", self.stop)
cherrypy.engine.subscribe("start", self.start)
def status(self):
"""Get the processing status. Returns time of last successful full
synchronisation with CERN/LDAP."""
with self._cv:
return self._full[:]
def stop(self, *args):
"""Tell the task thread to quit."""
with self._cv:
self._stopme = True
self._cv.notifyAll()
def run(self):
"""Run synchronisation thread."""
while True:
now = time.time()
until = now + self._interval
try:
self._sync(now)
except Exception as e:
cherrypy.log("SYNC ERROR %s.%s LDAP sync failed %s" %
(getattr(e, "__module__", "__builtins__"),
e.__class__.__name__, str(e)))
for line in format_exc().rstrip().split("\n"):
cherrypy.log(" " + line)
with self._cv:
while not self._stopme and now < until:
self._cv.wait(until - now)
now = time.time()
if self._stopme:
return
def _validate(self, input, type, regexp, now):
"""Convenience method to validate ldap data"""
if isinstance(input, type):
m = regexp.match(input)
if m: return m
if input not in self._warnings:
#Printing to logs only ascii encoded string
try:
input.decode('ascii')
except UnicodeDecodeError:
cherrypy.log(
"WARNING: ldap data failed validation, binary input, skipped printing."
)
else:
cherrypy.log("WARNING: ldap data failed validation: '%s'" %
input[:1000])
self._warnings[input] = now
return None
def _sync(self, now):
"""Perform full synchronisation."""
# Delete warnings older than 24 hours
for k, v in self._warnings.items():
if v < now - 86400:
del self._warnings[k]
# Get the user information from CERN/LDAP
ldresult = self._ldget(self._baseurl)
# Process each user record returned
rows = []
for (dn, attrs) in ldresult:
u = {
'username': attrs['sAMAccountName'][0],
'passwd': 'NeedsToBeUpdated',
'dn': dn,
'name': attrs['displayName'][0],
'email': attrs['mail'][0]
}
perid = attrs['employeeID'][0]
accstatus = attrs['userAccountControl'][0]
# Do the input validation
if not ( self._validate(u['username'], str, RX_USER, now) and \
self._validate(u['name'], basestring, RX_NAME, now) and \
self._validate(perid, str, RX_UID, now) and \
self._validate(u['dn'], basestring, RX_LDAPDN, now) and \
self._validate(u['email'], str, RX_EMAIL, now) and \
self._validate(accstatus, str, RX_UID, now) ):
cherrypy.log('WARNING: ignoring user with invalid non-optional ldap' \
' data: %s' % u['username'])
continue
# Only process normal accounts (aka enabled user accounts).
if accstatus == '512':
# newdn is the reversed elements from full name + personid + dn
newdn = ','.join(('CN=' + u['name'] + ',CN=' + perid + ',' +
u['dn']).split(',')[::-1])
# in case non-Cern DN was mapped to the account, use it as newdn instead
if 'altSecurityIdentities' in attrs.keys():
for altdn in attrs['altSecurityIdentities']:
m = self._validate(altdn, basestring, self.RX_ALTDN,
now)
if not m: continue
# get the last mapped DN not matching the Kerberosservice|CAs
newdn = m.group(1)
u['dn'] = '/' + newdn.replace(',', '/')
# add this user to the bulk data to be updated
rows.append(u)
# check number of rows is sane
if len(rows) < self._minreq:
cherrypy.log(
"ERROR: cowardly refusing full ldap synchronisation"
" with only %d users received, fewer than required %d" %
(len(rows), self._minreq))
return
cherrypy.log("INFO: found %d valid users in ldap" % len(rows))
# do the internal api call for the bulk update
result = []
self._intreq.put(("PUT", rows, result))
self._intreq.process()
self._full = (now, rows, ldresult, result and result[0])
def _ldget(self, url):
"""Get data from LDAP."""
result = []
ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, self._cacertdir)
l = ldap.initialize(url)
l.protocol_version = ldap.VERSION3
# Fetch paged results from ldap server.
# This is needed because there is a size limit on the CERN ldap server
# side to return at most 1000 entries per request.
# For more information, see http://tools.ietf.org/html/rfc2696.html
srv_ctrls = [
ldap.controls.SimplePagedResultsControl(criticality=False,
cookie="")
]
while True:
srv_ctrls[
0].size = 1000 # dont necessarily need to match the server limit
s = l.search_ext(
'OU=Users,OU=Organic Units,DC=cern,DC=ch',
ldap.SCOPE_SUBTREE,
'(memberOf:1.2.840.113556.1.4.1941:=CN=cms-authorized-users,OU=e-groups,OU=Workgroups,DC=cern,DC=ch)',
[
'sAMAccountName', 'displayName', 'employeeID', 'mail',
'altSecurityIdentities', 'userAccountControl'
],
serverctrls=srv_ctrls,
sizelimit=0)
_, res_data, _, srv_ctrls = l.result3(s, timeout=100)
result.extend(res_data)
if not srv_ctrls[0].cookie: break
if not result:
raise RuntimeError("Ldap returned no data for %s" % url)
return result
def _handle_init(self, c):
"""Initialise curl handle `c`."""
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
def _respond(self, c):
"""Respond to data on curl handle `c`."""
code = c.getinfo(pycurl.HTTP_CODE)
if code != 200:
raise RuntimeError("HTTP status %d for %s" %
(code, c.getinfo(pycurl.EFFECTIVE_URL)))
c.result.append(c.buffer.getvalue())
def _int_init(self, c, method, rows, result):
"""Initialise curl handle `c` for an internal REST API request."""
type, body = self._encode(rows)
headers = self._headers[:] + [("Content-Type", type),
("Content-Length", str(len(body)))]
if method == "PUT":
c.setopt(pycurl.POST, 0)
c.setopt(pycurl.UPLOAD, 1)
else:
assert False, "Unsupported method"
c.setopt(pycurl.URL, self._inturl)
c.setopt(pycurl.HTTPHEADER, ["%s: %s" % h for h in headers])
c.setopt(pycurl.READFUNCTION, StringIO(body).read)
c.result = result
def _encode(self, rows):
"""Encode dictionaries in `rows` for POST/PUT body as a HTML form."""
body, sep = "", ""
for obj in rows:
for key, value in obj.iteritems():
body += "%s%s=%s" % (sep, key,
urllib.quote(value.encode("utf-8")))
sep = "&"
return ("application/x-www-form-urlencoded", body)
class LdapSyncThread(Thread):
"""A task thread to synchronise SiteDB from CERN/LDAP. This runs on
a single node only in the cluster."""
_baseurl = "ldaps://xldap.cern.ch:636"
_ident = "SiteDB/%s Python/%s" % \
(os.environ["SITEDB_VERSION"], ".".join(map(str, sys.version_info[:3])))
# The buggy ca.cern.ch user interface allows to put anything, so some users
# have uploaded CA certificates or even SSH keys. So try to ignore them.
RX_ALTDN = re.compile(r"(?iu)^X509:.*<S>(([A-Z]+=([-\w _@'.()/]+),?)*(?<!berosservice|CERN Root CA|on Authority|s,CN=lsfcert|s,CN=acronmc))$")
def __init__(self, app, baseurl, mount, cacertdir = "/etc/grid-security/certificates", minreq = 1000, interval = 300, instance = "test"):
Thread.__init__(self, name = "LdapSync")
self.sectoken = "".join(random.sample(string.letters, 30))
self._inturl = "http://localhost:%d%s/%s/ldapsync" % \
(app.srvconfig.port, mount, instance)
self._headers = \
fake_authz_headers(open(app.srvconfig.tools.cms_auth.key_file).read(),
method = "Internal", login = self.sectoken,
name = self.__class__.__name__, dn = None,
roles = {"Global Admin": {"group": ["global"]}}) \
+ [("Accept", "application/json")]
self._cv = Condition()
if isinstance(baseurl, str):
self._baseurl = baseurl
self._cacertdir = cacertdir
self._minreq = minreq
self._interval = interval
self._stopme = False
self._full = (0, [], [])
self._warnings = {}
self._intreq = RequestManager(num_connections = 2,
user_agent = self._ident,
handle_init = self._handle_init,
request_respond = self._respond,
request_init = self._int_init)
cherrypy.engine.subscribe("stop", self.stop)
cherrypy.engine.subscribe("start", self.start)
def status(self):
"""Get the processing status. Returns time of last successful full
synchronisation with CERN/LDAP."""
with self._cv:
return self._full[:]
def stop(self, *args):
"""Tell the task thread to quit."""
with self._cv:
self._stopme = True
self._cv.notifyAll()
def run(self):
"""Run synchronisation thread."""
while True:
now = time.time()
until = now + self._interval
try:
self._sync(now)
except Exception as e:
cherrypy.log("SYNC ERROR %s.%s LDAP sync failed %s"
% (getattr(e, "__module__", "__builtins__"),
e.__class__.__name__, str(e)))
for line in format_exc().rstrip().split("\n"):
cherrypy.log(" " + line)
with self._cv:
while not self._stopme and now < until:
self._cv.wait(until - now)
now = time.time()
if self._stopme:
return
def _validate(self, input, type, regexp, now):
"""Convenience method to validate ldap data"""
if isinstance(input, type):
m = regexp.match(input)
if m: return m
if input not in self._warnings:
#Printing to logs only ascii encoded string
try:
input.decode('ascii')
except UnicodeDecodeError:
cherrypy.log("WARNING: ldap data failed validation, binary input, skipped printing.")
else:
cherrypy.log("WARNING: ldap data failed validation: '%s'" % input[:1000])
self._warnings[input] = now
return None
def _sync(self, now):
"""Perform full synchronisation."""
# Delete warnings older than 24 hours
for k, v in self._warnings.items():
if v < now - 86400:
del self._warnings[k]
# Get the user information from CERN/LDAP
ldresult = self._ldget(self._baseurl)
# Process each user record returned
rows = []
for (dn, attrs) in ldresult:
u = { 'username': attrs['sAMAccountName'][0],
'passwd' : 'NeedsToBeUpdated',
'dn' : dn,
'name' : attrs['displayName'][0],
'email' : attrs['mail'][0] }
perid = attrs['employeeID'][0]
accstatus = attrs['userAccountControl'][0]
# Do the input validation
if not ( self._validate(u['username'], str, RX_USER, now) and \
self._validate(u['name'], basestring, RX_NAME, now) and \
self._validate(perid, str, RX_UID, now) and \
self._validate(u['dn'], basestring, RX_LDAPDN, now) and \
self._validate(u['email'], str, RX_EMAIL, now) and \
self._validate(accstatus, str, RX_UID, now) ):
cherrypy.log('WARNING: ignoring user with invalid non-optional ldap' \
' data: %s' % u['username'])
continue
# Only process normal accounts (aka enabled user accounts).
if accstatus == '512':
# newdn is the reversed elements from full name + personid + dn
newdn = ','.join(('CN='+ u['name'] +',CN='+ perid +','+ u['dn']).split(',')[::-1])
# in case non-Cern DN was mapped to the account, use it as newdn instead
if 'altSecurityIdentities' in attrs.keys():
for altdn in attrs['altSecurityIdentities']:
m = self._validate(altdn, basestring, self.RX_ALTDN, now)
if not m: continue
# get the last mapped DN not matching the Kerberosservice|CAs
newdn = m.group(1)
u['dn'] = '/'+newdn.replace(',','/')
# add this user to the bulk data to be updated
rows.append(u)
# check number of rows is sane
if len(rows) < self._minreq:
cherrypy.log("ERROR: cowardly refusing full ldap synchronisation"
" with only %d users received, fewer than required %d"
% (len(rows), self._minreq))
return
cherrypy.log("INFO: found %d valid users in ldap" % len(rows))
# do the internal api call for the bulk update
result = []
self._intreq.put(("PUT", rows, result))
self._intreq.process()
self._full = (now, rows, ldresult, result and result[0])
def _ldget(self, url):
"""Get data from LDAP."""
result = []
ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, self._cacertdir)
l = ldap.initialize(url)
l.protocol_version = ldap.VERSION3
# Fetch paged results from ldap server.
# This is needed because there is a size limit on the CERN ldap server
# side to return at most 1000 entries per request.
# For more information, see http://tools.ietf.org/html/rfc2696.html
srv_ctrls = [ldap.controls.SimplePagedResultsControl(criticality=False, cookie="")]
while True:
srv_ctrls[0].size = 1000 # dont necessarily need to match the server limit
s = l.search_ext('OU=Users,OU=Organic Units,DC=cern,DC=ch',
ldap.SCOPE_SUBTREE,
'(memberOf:1.2.840.113556.1.4.1941:=CN=cms-authorized-users,OU=e-groups,OU=Workgroups,DC=cern,DC=ch)',
['sAMAccountName','displayName','employeeID','mail','altSecurityIdentities','userAccountControl'],
serverctrls=srv_ctrls,
sizelimit=0)
_, res_data, _, srv_ctrls = l.result3(s, timeout=100)
result.extend(res_data)
if not srv_ctrls[0].cookie: break
if not result:
raise RuntimeError("Ldap returned no data for %s" % url)
return result
def _handle_init(self, c):
"""Initialise curl handle `c`."""
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
def _respond(self, c):
"""Respond to data on curl handle `c`."""
code = c.getinfo(pycurl.HTTP_CODE)
if code != 200:
raise RuntimeError("HTTP status %d for %s" % (code, c.getinfo(pycurl.EFFECTIVE_URL)))
c.result.append(c.buffer.getvalue())
def _int_init(self, c, method, rows, result):
"""Initialise curl handle `c` for an internal REST API request."""
type, body = self._encode(rows)
headers = self._headers[:] + [("Content-Type", type),
("Content-Length", str(len(body)))]
if method == "PUT":
c.setopt(pycurl.POST, 0)
c.setopt(pycurl.UPLOAD, 1)
else:
assert False, "Unsupported method"
c.setopt(pycurl.URL, self._inturl)
c.setopt(pycurl.HTTPHEADER, ["%s: %s" % h for h in headers])
c.setopt(pycurl.READFUNCTION, StringIO(body).read)
c.result = result
def _encode(self, rows):
"""Encode dictionaries in `rows` for POST/PUT body as a HTML form."""
body, sep = "", ""
for obj in rows:
for key, value in obj.iteritems():
body += "%s%s=%s" % (sep, key, urllib.quote(value.encode("utf-8")))
sep = "&"
return ("application/x-www-form-urlencoded", body)
class RebusFetchThread(Thread):
"""A task thread to synchronise federation pledges from REBUS. This runs on
a single node only in the cluster."""
_baseurl = "http://gstat-wlcg.cern.ch/apps/pledges/resources/"
_ident = "SiteDB/%s Python/%s" % \
(os.environ["SITEDB_VERSION"], ".".join(map(str, sys.version_info[:3])))
def __init__(self, app, baseurl, mount, cacertdir = "/etc/grid-security/certificates", minreq = 1000, interval = 300, instance = "test"):
Thread.__init__(self, name = "RebusFetch")
self.sectoken = "".join(random.sample(string.letters, 30))
self._inturl = "http://localhost:%d%s/%s/rebusfetch" % \
(app.srvconfig.port, mount, instance)
self._headers = \
fake_authz_headers(open(app.srvconfig.tools.cms_auth.key_file).read(),
method = "Internal", login = self.sectoken,
name = self.__class__.__name__, dn = None,
roles = {"Global Admin": {"group": ["global"]}}) \
+ [("Accept", "application/json")]
self._cv = Condition()
if isinstance(baseurl, str):
self._baseurl = baseurl
self._cacertdir = cacertdir
self._minreq = minreq
self._interval = interval
self._stopme = False
self._full = (0, [], [])
self._warnings = {}
self._intreq = RequestManager(num_connections = 2,
user_agent = self._ident,
handle_init = self._handle_init,
request_respond = self._respond,
request_init = self._int_init)
cherrypy.engine.subscribe("stop", self.stop)
cherrypy.engine.subscribe("start", self.start)
def status(self):
"""Get the processing status. Returns time of last successful full
synchronisation with REBUS."""
with self._cv:
return self._full[:]
def stop(self, *args):
"""Tell the task thread to quit."""
with self._cv:
self._stopme = True
self._cv.notifyAll()
def run(self):
"""Run synchronisation thread."""
while True:
now = time.time()
until = now + self._interval
try:
self._sync(now)
except Exception as e:
cherrypy.log("SYNC ERROR %s.%s REBUS sync failed %s"
% (getattr(e, "__module__", "__builtins__"),
e.__class__.__name__, str(e)))
for line in format_exc().rstrip().split("\n"):
cherrypy.log(" " + line)
with self._cv:
while not self._stopme and now < until:
self._cv.wait(until - now)
now = time.time()
if self._stopme:
return
def _validate(self, input, type, regexp, now):
"""Convenience method to validate ldap data"""
if isinstance(input, type):
m = regexp.match(input)
if m: return m
if input not in self._warnings:
cherrypy.log("WARNING: REBUS data failed validation: '%s'" % input)
self._warnings[input] = now
return None
def _sync(self, now):
"""Perform full synchronisation."""
# Delete warnings older than 24 hours
for k, v in self._warnings.items():
if v < now - 86400:
del self._warnings[k]
result = []
# Get the user information from CERN/LDAP
ldresult = self._ldget(self._baseurl)
# get data from oracle database
# Process each user record returned
rows = []
id = 0;
for name, values in ldresult.iteritems():
for year, val1 in values["pledges"].iteritems():
i = { 'name' : name, 'country' : values["country"], 'year' : str(year), 'cpu' : str(0), 'disk' : str(0), 'tape' : str(0)}
if 'CPU' in val1.keys():
i['cpu'] = str(val1['CPU']/float(1000))
if 'Disk' in val1.keys():
i['disk'] = str(val1['Disk'])
if 'Tape' in val1.keys():
i['tape'] = str(val1['Tape'])
rows.append(i);
gett = self._intreq.put(("PUT", rows, result))
gettt = self._intreq.process()
def _ldget(self, url):
"""Get data from REBUS."""
year_next = date.today().year + 2
result = self._read_rebus_data(2008, year_next);
return result
def _handle_init(self, c):
"""Initialise curl handle `c`."""
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
def _respond(self, c):
"""Respond to data on curl handle `c`."""
code = c.getinfo(pycurl.HTTP_CODE)
if code != 200:
raise RuntimeError("HTTP status %d for %s" % (code, c.getinfo(pycurl.EFFECTIVE_URL)))
c.result.append(c.buffer.getvalue())
def _int_init(self, c, method, rows, result):
"""Initialise curl handle `c` for an internal REST API request."""
if method == "PUT":
type, body = self._encode(rows)
headers = self._headers[:] + [("Content-Type", type),
("Content-Length", str(len(body)))]
c.setopt(pycurl.POST, 0)
c.setopt(pycurl.UPLOAD, 1)
c.setopt(pycurl.URL, self._inturl)
c.setopt(pycurl.HTTPHEADER, ["%s: %s" % h for h in headers])
c.setopt(pycurl.READFUNCTION, StringIO(body).read)
c.result = result
elif method == "GET":
headers = self._headers[:]
c.setopt(pycurl.URL, self._inturl)
c.setopt(pycurl.HTTPHEADER, ["%s: %s" % h for h in headers])
c.result = result
else:
assert False, "Unsupported method"
def _encode(self, rows):
"""Encode dictionaries in `rows` for POST/PUT body as a HTML form."""
body, sep = "", ""
for obj in rows:
for key, value in obj.iteritems():
body += "%s%s=%s" % (sep, key, urllib2.quote(value.encode("utf-8")))
sep = "&"
return ("application/x-www-form-urlencoded", body)
def _read_rebus_data(self, year, yearTo):
"""REBUS json data fetch from 2008 to year.now + 2. All data returned in dictionary."""
data = {};
for x in range(year, yearTo):
url = "http://wlcg-rebus.cern.ch/apps/pledges/resources/"+str(x)+"/all/json";
req = urllib2.Request(url)
opener = urllib2.build_opener()
f = opener.open(req)
pledges = json.loads(f.read())
for index, item in enumerate(pledges):
federname = item["Federation"];
cms = item["CMS"];
pledgetype = item["PledgeType"];
country = item["Country"];
if cms and pledgetype:
if federname in data.keys():
if x in data[federname]["pledges"].keys():
data[federname]["pledges"][x][pledgetype]= cms;
else:
data[federname]["pledges"][x] = {pledgetype : cms};
else:
data[federname] = {"country" : country, "pledges" : {}};
data[federname]["pledges"] = {x : {pledgetype : cms}};
return data;
