def verify_chain(self, trusted_certs = None):
# Verify a chain of certificates. Each certificate must be signed by
# the public key contained in it's parent. The chain is recursed
# until a certificate is found that is signed by a trusted root.
# verify expiration time
if self.cert.has_expired():
raise CertExpired(self.get_subject(), "client cert")
# if this cert is signed by a trusted_cert, then we are set
for trusted_cert in trusted_certs:
if self.is_signed_by_cert(trusted_cert):
logger.debug("Cert %s signed by trusted cert %s", self.get_subject(), trusted_cert.get_subject())
# verify expiration of trusted_cert ?
if not trusted_cert.cert.has_expired():
return trusted_cert
else:
logger.debug("Trusted cert %s is expired", trusted_cert.get_subject())
# if there is no parent, then no way to verify the chain
if not self.parent:
#print self.get_subject(), "has no parent"
raise CertMissingParent(self.get_subject())
# if it wasn't signed by the parent...
if not self.is_signed_by_cert(self.parent):
#print self.get_subject(), "is not signed by parent"
return CertNotSignedByParent(self.get_subject())
# if the parent isn't verified...
self.parent.verify_chain(trusted_certs)
return
def GETRequestToOARRestAPI(self, request, strval=None ,next_page=None, username = None ):
self.oarserver['uri'] = \
OARGETParser.OARrequests_uri_dict[request]['uri']
#Get job details with username
if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username:
self.oarserver['uri'] += OARGETParser.OARrequests_uri_dict[request]['owner'] + username
headers = {}
data = json.dumps({})
logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" %(request))
if strval:
self.oarserver['uri'] = self.oarserver['uri'].\
replace("id",str(strval))
if next_page:
self.oarserver['uri'] += next_page
if username:
headers['X-REMOTE_IDENT'] = username
print>>sys.stderr, " \r\n \t OARrestapi \tGETRequestToOARRestAPI %s" %( self.oarserver['uri'])
logger.debug("OARrestapi: \t GETRequestToOARRestAPI \
self.oarserver['uri'] %s strval %s" \
%(self.oarserver['uri'], strval))
try :
#seems that it does not work if we don't add this
headers['content-length'] = '0'
conn = HTTPConnection(self.oarserver['ip'], \
self.oarserver['port'])
conn.request("GET", self.oarserver['uri'], data, headers)
resp = ( conn.getresponse()).read()
conn.close()
except HTTPException, error :
logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " \
%(error))
def DeleteOneLease(self, lease_id, username):
"""
Deletes the lease with the specified lease_id and username on OAR by
posting a delete request to OAR.
:param lease_id: Reservation identifier.
:param username: user's iotlab login in LDAP.
:type lease_id: Depends on what tou are using, could be integer or
string
:type username: string
:returns: dictionary with the lease id and if delete has been successful
(True) or no (False)
:rtype: dict
"""
# Here delete the lease specified
answer = self.query_sites.delete_experiment(lease_id, username)
# If the username is not necessary to delete the lease, then you can
# remove it from the parameters, given that you propagate the changes
# Return delete status so that you know if the delete has been
# successuf or not
if answer['status'] is True:
ret = {lease_id: True}
else:
ret = {lease_id: False}
logger.debug("CORTEXLAB_API \DeleteOneLease lease_id %s \r\n answer %s \
username %s" % (lease_id, answer, username))
return ret
def init_or_upgrade(self):
# check if under version control, and initialize it otherwise
if self.current_version() is None:
before = "Unknown"
# can be either a very old version, or a fresh install
# for very old versions:
self.handle_old_releases()
# in any case, initialize db from current code and reflect in migrate
model.init_tables(self.engine)
code_version = migrate.version(self.repository)
migrate.version_control(self.url, self.repository, code_version)
after = "%s" % self.current_version()
logger.info("DBSchema : jumped to version %s" % (after))
else:
# use migrate in the usual way
before = "%s" % self.current_version()
migrate.upgrade(self.url, self.repository)
after = "%s" % self.current_version()
if before != after:
logger.info("DBSchema : upgraded version from %s to %s" %
(before, after))
else:
logger.debug(
"DBSchema : no change needed in db schema (%s==%s)" %
(before, after))
def get_peer(self, xrn):
hrn, hrn_type = urn_to_hrn(xrn)
#Does this slice belong to a local site or a peer senslab site?
peer = None
# get this slice's authority (site)
slice_authority = get_authority(hrn)
site_authority = slice_authority
# get this site's authority (sfa root authority or sub authority)
#site_authority = get_authority(slice_authority).lower()
logger.debug("SLABSLICES \ get_peer slice_authority %s \
site_authority %s hrn %s" %(slice_authority, \
site_authority, hrn))
#This slice belongs to the current site
if site_authority == self.driver.root_auth :
return None
# check if we are already peered with this site_authority, if so
#peers = self.driver.GetPeers({})
peers = self.driver.GetPeers(peer_filter = slice_authority)
for peer_record in peers:
if site_authority == peer_record.hrn:
peer = peer_record
logger.debug(" SLABSLICES \tget_peer peer %s " %(peer))
return peer
def sign(self):
logger.debug('certificate.sign')
assert self.cert != None
assert self.issuerSubject != None
assert self.issuerKey != None
self.cert.set_issuer(self.issuerSubject)
self.cert.sign(self.issuerKey.get_openssl_pkey(), self.digest)
def sign(self):
logger.debug('certificate.sign')
assert self.cert != None
assert self.issuerSubject != None
assert self.issuerKey != None
self.cert.set_issuer(self.issuerSubject)
self.cert.sign(self.issuerKey.get_openssl_pkey(), self.digest)
def __init__(self,
create=False,
subject=None,
string=None,
filename=None,
uuid=None,
hrn=None,
urn=None,
lifeDays=1825,
email=None):
self.uuid = None
self.hrn = None
self.urn = None
self.email = None # for adding to the SubjectAltName
Certificate.__init__(self, lifeDays, create, subject, string, filename)
if subject:
logger.debug("Creating GID for subject: %s" % subject)
if uuid:
self.uuid = int(uuid)
if hrn:
self.hrn = hrn
self.urn = hrn_to_urn(hrn, 'unknown')
if urn:
self.urn = urn
self.hrn, type = urn_to_hrn(urn)
if email:
self.set_email(email)
def verify_slice_nodes(self, sfa_slice, requested_slivers, peer):
current_slivers = []
deleted_nodes = []
if 'node_ids' in sfa_slice:
nodes = self.driver.GetNodes(sfa_slice['list_node_ids'], \
['hostname'])
current_slivers = [node['hostname'] for node in nodes]
# remove nodes not in rspec
deleted_nodes = list(set(current_slivers).\
difference(requested_slivers))
# add nodes from rspec
#added_nodes = list(set(requested_slivers).\
#difference(current_slivers))
logger.debug("SLABSLICES \tverify_slice_nodes slice %s\
\r\n \r\n deleted_nodes %s"\
%(sfa_slice, deleted_nodes))
if deleted_nodes:
#Delete the entire experience
self.driver.DeleteSliceFromNodes(sfa_slice)
#self.driver.DeleteSliceFromNodes(sfa_slice['slice_hrn'], \
#deleted_nodes)
return nodes
def DeleteSliceFromNodes(self, slice_record):
"""
Deletes all the running or scheduled jobs of a given slice
given its record.
:param slice_record: record of the slice, must contain oar_job_id, user
:type slice_record: dict
:returns: dict of the jobs'deletion status. Success= True, Failure=
False, for each job id.
:rtype: dict
"""
logger.debug("IOTLAB_API \t DeleteSliceFromNodes %s " %
(slice_record))
if isinstance(slice_record['oar_job_id'], list):
oar_bool_answer = {}
for job_id in slice_record['oar_job_id']:
ret = self.DeleteJobs(job_id, slice_record['user'])
oar_bool_answer.update(ret)
else:
oar_bool_answer = self.DeleteJobs(slice_record['oar_job_id'],
slice_record['user'])
return oar_bool_answer
def GetReservedNodes(self, username=None):
""" Get list of leases. Get the leases for the username if specified,
otherwise get all the leases. Finds the nodes hostnames for each
OAR node identifier.
:param username: user's LDAP login
:type username: string
:returns: list of reservations dict
:rtype: dict list
"""
#Get the nodes in use and the reserved nodes
reservation_dict_list = \
self.oar.parser.SendRequest("GET_reserved_nodes", \
username = username)
# Get the full node dict list once for all
# so that we can get the hostnames given their oar node id afterwards
# when the reservations are checked.
full_nodes_dict_list = self.GetNodes()
#Put the full node list into a dictionary keyed by oar node id
oar_id_node_dict = {}
for node in full_nodes_dict_list:
oar_id_node_dict[node['oar_id']] = node
for resa in reservation_dict_list:
logger.debug ("GetReservedNodes resa %s"%(resa))
#dict list of hostnames and their site
resa['reserved_nodes'] = \
self.__get_hostnames_from_oar_node_ids(oar_id_node_dict,
resa['resource_ids'])
#del resa['resource_ids']
return reservation_dict_list
def DeleteLeases(self, leases_id_list, slice_hrn):
"""
Deletes several leases, based on their job ids and the slice
they are associated with. Uses DeleteJobs to delete the jobs
on OAR. Note that one slice can contain multiple jobs, and in this
case all the jobs in the leases_id_list MUST belong to ONE slice,
since there is only one slice hrn provided here.
:param leases_id_list: list of job ids that belong to the slice whose
slice hrn is provided.
:param slice_hrn: the slice hrn.
:type slice_hrn: string
.. warning:: Does not have a return value since there was no easy
way to handle failure when dealing with multiple job delete. Plus,
there was no easy way to report it to the user.
"""
logger.debug("IOTLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
\r\n " % (leases_id_list, slice_hrn))
for job_id in leases_id_list:
self.DeleteJobs(job_id, slice_hrn)
return
def DeleteLeases(self, leases_id_list, slice_hrn):
"""
Deletes several leases, based on their experiment ids and the slice
they are associated with. Uses DeleteOneLease to delete the
experiment on the testbed. Note that one slice can contain multiple
experiments, and in this
case all the experiments in the leases_id_list MUST belong to this
same slice, since there is only one slice hrn provided here.
:param leases_id_list: list of job ids that belong to the slice whose
slice hrn is provided.
:param slice_hrn: the slice hrn.
:type slice_hrn: string
.. warning:: Does not have a return value since there was no easy
way to handle failure when dealing with multiple job delete. Plus,
there was no easy way to report it to the user.
"""
logger.debug("CORTEXLAB_API DeleteLeases leases_id_list %s slice_hrn %s \
\r\n " %(leases_id_list, slice_hrn))
for experiment_id in leases_id_list:
self.DeleteOneLease(experiment_id, slice_hrn)
return
def DeleteJobs(self, job_id, username):
"""
Deletes the job with the specified job_id and username on OAR by
posting a delete request to OAR.
:param job_id: job id in OAR.
:param username: user's iotlab login in LDAP.
:type job_id: integer
:type username: string
:returns: dictionary with the job id and if delete has been successful
(True) or no (False)
:rtype: dict
"""
logger.debug("IOTLAB_API \tDeleteJobs jobid %s username %s " %
(job_id, username))
if not job_id or job_id is -1:
return
reqdict = {}
reqdict['method'] = "delete"
reqdict['strval'] = str(job_id)
answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id', reqdict,
username)
if answer['status'] == 'Delete request registered':
ret = {job_id: True}
else:
ret = {job_id: False}
logger.debug("IOTLAB_API \tDeleteJobs jobid %s \r\n answer %s \
username %s" % (job_id, answer, username))
return ret
def GetReservedNodes(self, username=None):
""" Get list of leases. Get the leases for the username if specified,
otherwise get all the leases. Finds the nodes hostnames for each
OAR node identifier.
:param username: user's LDAP login
:type username: string
:returns: list of reservations dict
:rtype: dict list
"""
#Get the nodes in use and the reserved nodes
reservation_dict_list = \
self.oar.parser.SendRequest("GET_reserved_nodes", \
username = username)
# Get the full node dict list once for all
# so that we can get the hostnames given their oar node id afterwards
# when the reservations are checked.
full_nodes_dict_list = self.GetNodes()
#Put the full node list into a dictionary keyed by oar node id
oar_id_node_dict = {}
for node in full_nodes_dict_list:
oar_id_node_dict[node['oar_id']] = node
for resa in reservation_dict_list:
logger.debug("GetReservedNodes resa %s" % (resa))
#dict list of hostnames and their site
resa['reserved_nodes'] = \
self.__get_hostnames_from_oar_node_ids(oar_id_node_dict,
resa['resource_ids'])
#del resa['resource_ids']
return reservation_dict_list
def __init__(self, server_address, HandlerClass, key_file, cert_file, logRequests=True):
"""Secure XML-RPC server.
It it very similar to SimpleXMLRPCServer but it uses HTTPS for transporting XML data.
"""
logger.debug("SecureXMLRPCServer.__init__, server_address=%s, cert_file=%s"%(server_address,cert_file))
self.logRequests = logRequests
self.interface = None
self.key_file = key_file
self.cert_file = cert_file
self.method_map = {}
# add cache to the request handler
HandlerClass.cache = Cache()
#for compatibility with python 2.4 (centos53)
if sys.version_info < (2, 5):
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self)
else:
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, True, None)
SocketServer.BaseServer.__init__(self, server_address, HandlerClass)
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_privatekey_file(key_file)
ctx.use_certificate_file(cert_file)
# If you wanted to verify certs against known CAs.. this is how you would do it
#ctx.load_verify_locations('/etc/sfa/trusted_roots/plc.gpo.gid')
config = Config()
trusted_cert_files = TrustedRoots(config.get_trustedroots_dir()).get_file_list()
for cert_file in trusted_cert_files:
ctx.load_verify_locations(cert_file)
ctx.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, verify_callback)
ctx.set_verify_depth(5)
ctx.set_app_data(self)
self.socket = SSL.Connection(ctx, socket.socket(self.address_family,
self.socket_type))
self.server_bind()
self.server_activate()
def get_slice_and_slivers(self, slice_xrn, login=None):
"""
Returns a dict of slivers keyed on the sliver's node_id
"""
slivers = {}
sfa_slice = None
if not slice_xrn:
return (sfa_slice, slivers)
slice_urn = hrn_to_urn(slice_xrn, 'slice')
slice_hrn, _ = urn_to_hrn(slice_xrn)
slice_name = slice_hrn
slices = self.driver.GetSlices(slice_filter= str(slice_name), \
slice_filter_type = 'slice_hrn', login=login)
logger.debug("Slabaggregate api \tget_slice_and_slivers \
sfa_slice %s \r\n slices %s self.driver.hrn %s" \
%(sfa_slice, slices, self.driver.hrn))
if not slices:
return (sfa_slice, slivers)
#if isinstance(sfa_slice, list):
#sfa_slice = slices[0]
#else:
#sfa_slice = slices
# sort slivers by node id , if there is a job
#and therfore, node allocated to this slice
for sfa_slice in slices:
try:
node_ids_list = sfa_slice['node_ids']
except KeyError:
logger.log_exc("SLABAGGREGATE \t \
get_slice_and_slivers KeyError ")
continue
for node in node_ids_list:
sliver_xrn = Xrn(slice_urn, type='sliver', id=node)
sliver_xrn.set_authority(self.driver.hrn)
#node_id = self.driver.root_auth + '.' + node_id
sliver = Sliver({'sliver_id':sliver_xrn.urn,
'name': sfa_slice['hrn'],
'type': 'slab-node',
'tags': []})
slivers[node] = sliver
#Add default sliver attribute :
#connection information for senslab
if get_authority (sfa_slice['hrn']) == self.driver.root_auth:
tmp = sfa_slice['hrn'].split('.')
ldap_username = tmp[1].split('_')[0]
vmaddr = 'ssh ' + ldap_username + '@grenoble.senslab.info'
slivers['default_sliver'] = {'vm': vmaddr , 'login': ldap_username}
#TODO get_slice_and_slivers Find the login of the external user
logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\
%(slivers))
return (slices, slivers)
def DeleteSliceFromNodes(self, slice_record):
"""
Deletes all the running or scheduled jobs of a given slice
given its record.
:param slice_record: record of the slice, must contain experiment_id,
user
:type slice_record: dict
:returns: dict of the jobs'deletion status. Success= True, Failure=
False, for each job id.
:rtype: dict
.. note: used in driver delete_sliver
"""
logger.debug("CORTEXLAB_API \t DeleteSliceFromNodes %s "
% (slice_record))
if isinstance(slice_record['experiment_id'], list):
experiment_bool_answer = {}
for experiment_id in slice_record['experiment_id']:
ret = self.DeleteOneLease(experiment_id, slice_record['user'])
experiment_bool_answer.update(ret)
else:
experiment_bool_answer = [self.DeleteOneLease(
slice_record['experiment_id'],
slice_record['user'])]
return experiment_bool_answer
def merge_node(self, source_node_tag, network, no_dupes = False):
logger.debug("SLABV1 merge_node")
#if no_dupes and self.get_node_element(node['hostname']):
## node already exists
#return
network_tag = self.add_network(network)
network_tag.append(deepcopy(source_node_tag))
def LdapDeleteUser(self, record_filter):
"""Deletes a SFA person in LDAP, based on the user's hrn.
:param record_filter: Filter to find the user to be deleted. Must
contain at least the user's email.
:type record_filter: dict
:returns: dict with bool True if successful, bool False and error
message otherwise.
:rtype: dict
.. seealso:: LdapFindUser docstring for more info on record filter.
.. seealso:: LdapDelete for user deletion
"""
#Find uid of the person
person = self.LdapFindUser(record_filter, [])
logger.debug("LDAPapi.py \t LdapDeleteUser record %s person %s"
% (record_filter, person))
if person:
dn = 'uid=' + person['uid'] + "," + self.baseDN
else:
return {'bool': False}
result = self.LdapDelete(dn)
return result
def __init__ (self, config):
dbname="sfa"
# will be created lazily on-demand
self._session = None
# the former PostgreSQL.py used the psycopg2 directly and was doing
#self.connection.set_client_encoding("UNICODE")
# it's unclear how to achieve this in sqlalchemy, nor if it's needed at all
# http://www.sqlalchemy.org/docs/dialects/postgresql.html#unicode
# we indeed have /var/lib/pgsql/data/postgresql.conf where
# this setting is unset, it might be an angle to tweak that if need be
# try a unix socket first - omitting the hostname does the trick
unix_url = "postgresql+psycopg2://%s:%s@:%s/%s"%\
(config.SFA_DB_USER,config.SFA_DB_PASSWORD,config.SFA_DB_PORT,dbname)
# the TCP fallback method
tcp_url = "postgresql+psycopg2://%s:%s@%s:%s/%s"%\
(config.SFA_DB_USER,config.SFA_DB_PASSWORD,config.SFA_DB_HOST,config.SFA_DB_PORT,dbname)
for url in [ unix_url, tcp_url ] :
try:
logger.debug("Trying db URL %s"%url)
self.engine = create_engine (url)
self.check()
self.url=url
return
except:
pass
self.engine=None
raise Exception,"Could not connect to database %s as %s with psycopg2"%(dbname,config.SFA_DB_USER)
def list_slices(self, creds, options):
# look in cache first
if self.cache:
slices = self.cache.get('slices')
if slices:
logger.debug("NitosDriver.list_slices returns from cache")
return slices
# get data from db
slices = self.shell.getSlices({}, [])
testbed_name = self.testbedInfo['name']
slice_hrns = [
slicename_to_hrn(self.hrn, testbed_name, slice['slice_name'])
for slice in slices
]
slice_urns = [
hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns
]
# cache the result
if self.cache:
logger.debug("NitosDriver.list_slices stores value in cache")
self.cache.add('slices', slice_urns)
return slice_urns
def update_jobs_in_iotlabdb(self, job_oar_list, jobs_psql):
""" Cleans the iotlab db by deleting expired and cancelled jobs.
Compares the list of job ids given by OAR with the job ids that
are already in the database, deletes the jobs that are no longer in
the OAR job id list.
:param job_oar_list: list of job ids coming from OAR
:type job_oar_list: list
:param job_psql: list of job ids from the database.
:type job_psql: list
:returns: None
"""
#Turn the list into a set
set_jobs_psql = set(jobs_psql)
kept_jobs = set(job_oar_list).intersection(set_jobs_psql)
logger.debug("\r\n \t update_jobs_in_iotlabdb jobs_psql %s \r\n \
job_oar_list %s kept_jobs %s " %
(set_jobs_psql, job_oar_list, kept_jobs))
deleted_jobs = set_jobs_psql.difference(kept_jobs)
deleted_jobs = list(deleted_jobs)
if len(deleted_jobs) > 0:
self.iotlab_session.query(IotlabXP).filter(
IotlabXP.job_id.in_(deleted_jobs)).delete(
synchronize_session='fetch')
self.iotlab_session.commit()
return
def list_resources (self, slice_urn, slice_hrn, creds, options):
# right now rspec_version is ignored on the federica side
# we normally derive it from options
# look in cache if client has requested so
cached_requested = options.get('cached', True)
# global advertisement
if not slice_hrn:
# self.cache is initialized unless the global config has it turned off
if cached_requested and self.cache:
# using federica_version_string as the key into the cache
rspec = self.cache.get(federica_version_string)
if rspec:
logger.debug("FdDriver.ListResources: returning cached advertisement")
return self.response(rspec)
# otherwise, need to get it
# java code expects creds as a String
# rspec = self.shell.listAvailableResources (creds, federica_version_string)
rspec = self.shell.listAvailableResources ("", federica_version_string)
# rspec = self.shell.listAvailableResources (federica_version_string)
# cache it for future use
if self.cache:
logger.debug("FdDriver.ListResources: stores advertisement in cache")
self.cache.add(federica_version_string, rspec)
return self.response(rspec)
# about a given slice : don't cache
else:
# java code expects creds as a String
# return self.response(self.shell.listSliceResources(creds, federica_version_string, slice_urn))
return self.response(self.shell.listSliceResources("", federica_version_string, slice_urn))
def list_resources (self, slice_urn, slice_hrn, creds, options):
cached_requested = options.get('cached', True)
version_manager = VersionManager()
# get the rspec's return format from options
rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
version_string = "rspec_%s" % (rspec_version)
#panos adding the info option to the caching key (can be improved)
if options.get('info'):
version_string = version_string + "_"+options.get('info', 'default')
# look in cache first
if cached_requested and self.cache and not slice_hrn:
rspec = self.cache.get(version_string)
if rspec:
logger.debug("OpenStackDriver.ListResources: returning cached advertisement")
return rspec
#panos: passing user-defined options
#print "manager options = ",options
aggregate = OSAggregate(self)
rspec = aggregate.get_rspec(slice_xrn=slice_urn, version=rspec_version,
options=options)
# cache the result
if self.cache and not slice_hrn:
logger.debug("OpenStackDriver.ListResources: stores advertisement in cache")
self.cache.add(version_string, rspec)
return rspec
def list_resources(self, slice_urn, slice_hrn, creds, options):
# right now rspec_version is ignored on the federica side
# we normally derive it from options
# look in cache if client has requested so
cached_requested = options.get('cached', True)
# global advertisement
if not slice_hrn:
# self.cache is initialized unless the global config has it turned off
if cached_requested and self.cache:
# using federica_version_string as the key into the cache
rspec = self.cache.get(federica_version_string)
if rspec:
logger.debug(
"FdDriver.ListResources: returning cached advertisement"
)
return self.response(rspec)
# otherwise, need to get it
# java code expects creds as a String
# rspec = self.shell.listAvailableResources (creds, federica_version_string)
rspec = self.shell.listAvailableResources("",
federica_version_string)
# rspec = self.shell.listAvailableResources (federica_version_string)
# cache it for future use
if self.cache:
logger.debug(
"FdDriver.ListResources: stores advertisement in cache")
self.cache.add(federica_version_string, rspec)
return self.response(rspec)
# about a given slice : don't cache
else:
# java code expects creds as a String
# return self.response(self.shell.listSliceResources(creds, federica_version_string, slice_urn))
return self.response(
self.shell.listSliceResources("", federica_version_string,
slice_urn))
def find_max_uidNumber(self):
"""Find the LDAP max uidNumber (POSIX uid attribute).
Used when adding a new user in LDAP Directory
:returns: max uidNumber + 1
:rtype: string
"""
#First, get all the users in the LDAP
get_attrs = "(uidNumber=*)"
login_filter = ['uidNumber']
result_data = self.LdapSearch(get_attrs, login_filter)
#It there is no user in LDAP yet, First LDAP user
if result_data == []:
max_uidnumber = self.ldapUserUidNumberMin
#Otherwise, get the highest uidNumber
else:
uidNumberList = [int(r[1]['uidNumber'][0])for r in result_data]
logger.debug("LDAPapi.py \tfind_max_uidNumber \
uidNumberList %s " % (uidNumberList))
max_uidnumber = max(uidNumberList) + 1
return str(max_uidnumber)
def verify_chain(self, trusted_certs=None):
# Verify a chain of certificates. Each certificate must be signed by
# the public key contained in it's parent. The chain is recursed
# until a certificate is found that is signed by a trusted root.
# verify expiration time
if self.cert.has_expired():
raise CertExpired(self.get_subject(), "client cert")
# if this cert is signed by a trusted_cert, then we are set
for trusted_cert in trusted_certs:
if self.is_signed_by_cert(trusted_cert):
logger.debug("Cert %s signed by trusted cert %s",
self.get_subject(), trusted_cert.get_subject())
# verify expiration of trusted_cert ?
if not trusted_cert.cert.has_expired():
return trusted_cert
else:
logger.debug("Trusted cert %s is expired",
trusted_cert.get_subject())
# if there is no parent, then no way to verify the chain
if not self.parent:
#print self.get_subject(), "has no parent"
raise CertMissingParent(self.get_subject())
# if it wasn't signed by the parent...
if not self.is_signed_by_cert(self.parent):
#print self.get_subject(), "is not signed by parent"
return CertNotSignedByParent(self.get_subject())
# if the parent isn't verified...
self.parent.verify_chain(trusted_certs)
return
def DeleteSliceFromNodes(self, slice_record):
"""
Deletes all the running or scheduled jobs of a given slice
given its record.
:param slice_record: record of the slice, must contain oar_job_id, user
:type slice_record: dict
:returns: dict of the jobs'deletion status. Success= True, Failure=
False, for each job id.
:rtype: dict
"""
logger.debug("IOTLAB_API \t DeleteSliceFromNodes %s "
% (slice_record))
if isinstance(slice_record['oar_job_id'], list):
oar_bool_answer = {}
for job_id in slice_record['oar_job_id']:
ret = self.DeleteJobs(job_id, slice_record['user'])
oar_bool_answer.update(ret)
else:
oar_bool_answer = self.DeleteJobs(slice_record['oar_job_id'],
slice_record['user'])
return oar_bool_answer
def func(*args, **kwds):
if not name in DummyShell.direct_calls:
raise Exception, "Illegal method call %s for DUMMY driver" % (
name)
result = getattr(self.proxy, name)(*args, **kwds)
logger.debug('DummyShell %s returned ... ' % (name))
return result
def merge_node(self, source_node_tag, network, no_dupes=False):
logger.debug("SLABV1 merge_node")
#if no_dupes and self.get_node_element(node['hostname']):
## node already exists
#return
network_tag = self.add_network(network)
network_tag.append(deepcopy(source_node_tag))
def DeleteJobs(self, job_id, username):
"""
Deletes the job with the specified job_id and username on OAR by
posting a delete request to OAR.
:param job_id: job id in OAR.
:param username: user's iotlab login in LDAP.
:type job_id: integer
:type username: string
:returns: dictionary with the job id and if delete has been successful
(True) or no (False)
:rtype: dict
"""
logger.debug("IOTLAB_API \tDeleteJobs jobid %s username %s "
% (job_id, username))
if not job_id or job_id is -1:
return
reqdict = {}
reqdict['method'] = "delete"
reqdict['strval'] = str(job_id)
answer = self.oar.POSTRequestToOARRestAPI('DELETE_jobs_id',
reqdict, username)
if answer['status'] == 'Delete request registered':
ret = {job_id: True}
else:
ret = {job_id: False}
logger.debug("IOTLAB_API \tDeleteJobs jobid %s \r\n answer %s \
username %s" % (job_id, answer, username))
return ret
def _process_ldap_info_for_one_user(self, record, result_data):
"""
Put the user's ldap data into shape. Only deals with one user
record and one user data from ldap.
:param record: user record
:param result_data: Raw ldap data coming from LdapSearch
:returns: user's data dict with 'type','pkey','uid', 'email',
'first_name' 'last_name''serial''authority''peer_authority'
'pointer''hrn'
:type record: dict
:type result_data: list
:rtype :dict
"""
#One entry only in the ldap data because we used a filter
#to find one user only
ldapentry = result_data[0][1]
logger.debug("LDAP.PY \t LdapFindUser ldapentry %s" % (ldapentry))
tmpname = ldapentry['uid'][0]
tmpemail = ldapentry['mail'][0]
if ldapentry['mail'][0] == "unknown":
tmpemail = None
parent_hrn = None
peer_authority = None
if 'hrn' in record:
hrn = record['hrn']
parent_hrn = get_authority(hrn)
if parent_hrn != self.authname:
peer_authority = parent_hrn
#In case the user was not imported from Iotlab LDAP
#but from another federated site, has an account in
#iotlab but currently using his hrn from federated site
#then the login is different from the one found in its hrn
if tmpname != hrn.split('.')[1]:
hrn = None
else:
hrn = None
results = {
'type': 'user',
'pkey': ldapentry['sshPublicKey'],
#'uid': ldapentry[1]['uid'][0],
'uid': tmpname,
'email': tmpemail,
#'email': ldapentry[1]['mail'][0],
'first_name': ldapentry['givenName'][0],
'last_name': ldapentry['sn'][0],
#'phone': 'none',
'serial': 'none',
'authority': parent_hrn,
'peer_authority': peer_authority,
'pointer': -1,
'hrn': hrn,
}
return results
def LdapModifyUser(self, user_record, new_attributes_dict):
"""
Gets the record from one user based on the user sfa recordand changes
the attributes according to the specified new_attributes. Do not use
this if we need to modify the uid. Use a ModRDN operation instead
( modify relative DN ).
:param user_record: sfa user record.
:param new_attributes_dict: new user attributes, keys must be the
same as the LDAP model.
:type user_record: dict
:type new_attributes_dict: dict
:returns: bool True if successful, bool False if not.
:rtype: dict
.. seealso:: make_ldap_filters_from_record for info on what is mandatory
in the user_record.
.. seealso:: make_ldap_attributes_from_record for the LDAP objectclass.
"""
if user_record is None:
logger.error("LDAP \t LdapModifyUser Need user record ")
return {'bool': False}
#Get all the attributes of the user_uid_login
#person = self.LdapFindUser(record_filter,[])
req_ldap = self.make_ldap_filters_from_record(user_record)
person_list = self.LdapSearch(req_ldap, [])
logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s"
% (person_list))
if person_list and len(person_list) > 1:
logger.error("LDAP \t LdapModifyUser Too many users returned")
return {'bool': False}
if person_list is None:
logger.error("LDAP \t LdapModifyUser User %s doesn't exist "
% (user_record))
return {'bool': False}
# The dn of our existing entry/object
#One result only from ldapSearch
person = person_list[0][1]
dn = 'uid=' + person['uid'][0] + "," + self.baseDN
if new_attributes_dict:
old = {}
for k in new_attributes_dict:
if k not in person:
old[k] = ''
else:
old[k] = person[k]
logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"
% (new_attributes_dict))
result = self.LdapModify(dn, old, new_attributes_dict)
return result
else:
logger.error("LDAP \t LdapModifyUser No new attributes given. ")
return {'bool': False}
def get_nodes_with_slivers(xml, sliver_filter={}):
xpath = '//node[count(sliver)>0] | \
//default:node[count(default:sliver) > 0]'
node_elems = xml.xpath(xpath)
logger.debug("SLABV1NODE \tget_nodes_with_slivers \
node_elems %s"%(node_elems))
return Iotlabv1Node.get_node_objs(node_elems)
def get_nodes_with_slivers(xml, sliver_filter={}):
xpath = '//node[count(sliver)>0] | \
//default:node[count(default:sliver) > 0]'
node_elems = xml.xpath(xpath)
logger.debug("SLABV1NODE \tget_nodes_with_slivers \
node_elems %s"%(node_elems))
return Slabv1Node.get_node_objs(node_elems)
def init_self_signed_cert(hrn, key, server_cert_file):
logger.debug("generating self signed cert")
# generate self signed certificate
cert = Certificate(subject=hrn)
cert.set_issuer(key=key, subject=hrn)
cert.set_pubkey(key)
cert.sign()
cert.save_to_file(server_cert_file)
def _create_job_structure_request_for_OAR(lease_dict):
""" Creates the structure needed for a correct POST on OAR.
Makes the timestamp transformation into the appropriate format.
Sends the POST request to create the job with the resources in
added_nodes.
"""
nodeid_list = []
reqdict = {}
reqdict['workdir'] = '/tmp'
reqdict['resource'] = "{network_address in ("
for node in lease_dict['added_nodes']:
logger.debug("\r\n \r\n OARrestapi \t \
__create_job_structure_request_for_OAR node %s" %(node))
# Get the ID of the node
nodeid = node
reqdict['resource'] += "'" + nodeid + "', "
nodeid_list.append(nodeid)
custom_length = len(reqdict['resource'])- 2
reqdict['resource'] = reqdict['resource'][0:custom_length] + \
")}/nodes=" + str(len(nodeid_list))
walltime, sleep_walltime = \
IotlabShell._process_walltime(\
int(lease_dict['lease_duration']))
reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
":" + str(walltime[1]) + ":" + str(walltime[2])
reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
#In case of a scheduled experiment (not immediate)
#To run an XP immediately, don't specify date and time in RSpec
#They will be set to None.
if lease_dict['lease_start_time'] is not '0':
#Readable time accepted by OAR
# converting timestamp to date in the local timezone tz = None
start_time = datetime.fromtimestamp( \
int(lease_dict['lease_start_time']), tz=None).\
strftime(lease_dict['time_format'])
reqdict['reservation'] = str(start_time)
#If there is not start time, Immediate XP. No need to add special
# OAR parameters
reqdict['type'] = "deploy"
reqdict['directory'] = ""
reqdict['name'] = "SFA_" + lease_dict['slice_user']
return reqdict
def load (self):
try:
infile=file(self.filename,'r')
self.url2version=pickle.load(infile)
infile.close()
except:
logger.debug("Cannot load version cache, restarting from scratch")
self.url2version = {}
logger.debug("loaded version cache with %d entries %s"%(len(self.url2version),self.url2version.keys()))
def func(*args, **kwds):
actual_name=None
if name in NitosShell.direct_calls: actual_name=name
if not actual_name:
raise Exception, "Illegal method call %s for NITOS driver"%(name)
actual_name = "scheduler.server." + actual_name
result=getattr(self.proxy, actual_name)(*args, **kwds)
logger.debug('NitosShell %s (%s) returned ... '%(name,actual_name))
return result
def _dispatch(self, method, params):
logger.debug("SecureXMLRPCServer._dispatch, method=%s"%method)
try:
return SimpleXMLRPCServer.SimpleXMLRPCDispatcher._dispatch(self, method, params)
except:
# can't use format_exc() as it is not available in jython yet
# (even in trunk).
type, value, tb = sys.exc_info()
raise xmlrpclib.Fault(1,''.join(traceback.format_exception(type, value, tb)))
def POSTRequestToOARRestAPI(self, request, datadict, username=None):
""" Used to post a job on OAR , along with data associated
with the job.
"""
#first check that all params for are OK
try:
self.oarserver['uri'] = \
self.OAR_REQUEST_POST_URI_DICT[request]['uri']
except KeyError:
logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \
valid")
return
if datadict and 'strval' in datadict:
self.oarserver['uri'] = self.oarserver['uri'].replace("id", \
str(datadict['strval']))
del datadict['strval']
data = json.dumps(datadict)
headers = {'X-REMOTE_IDENT':username, \
'content-type': self.POST_FORMAT['json']['content'], \
'content-length':str(len(data))}
try :
conn = HTTPConnection(self.oarserver['ip'], \
self.oarserver['port'])
conn.request("POST", self.oarserver['uri'], data, headers)
resp = conn.getresponse()
body = resp.read()
except NotConnected:
logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \
data %s \r\n \t\n \t\t headers %s uri %s" \
%(data,headers,self.oarserver['uri']))
except Exception as error:
logger.log_exc("POST_OAR_SERVER : Connection error: %s "
% (error))
raise Exception ("POST_OAR_SERVER : Connection error %s " %(error))
finally:
conn.close()
if resp.status >= 400:
raise ValueError ("Response Error %s, %s" %(resp.status,
resp.reason))
try:
answer = json.loads(body)
logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer))
return answer
except ValueError, error:
logger.log_exc("Failed to parse Server Response: error %s \
%s" %(error))
def func(*args, **kwds):
actual_name=None
if name in PlShell.direct_calls: actual_name=name
if name in PlShell.alias_calls: actual_name=PlShell.alias_calls[name]
if not actual_name:
raise Exception, "Illegal method call %s for PL driver"%(name)
result=getattr(self.proxy, actual_name)(self.plauth, *args, **kwds)
logger.debug('PlShell %s (%s) returned ... '%(name,actual_name))
return result
def func(*args, **kwds):
if name not in FdShell.direct_calls:
raise Exception, "Illegal method call %s for FEDERICA driver"%(name)
logger.info("Issuing %s args=%s kwds=%s to federica"%\
(name,args,kwds))
# result=getattr(self.proxy, "AggregateManager.%s"%name)(credential, *args, **kwds)
result=getattr(self.proxy, "AggregateManager.%s"%name)(*args, **kwds)
logger.debug('FdShell %s (%s) returned ... '%(name,name))
return result
def LdapSearch(self, req_ldap=None, expected_fields=None):
"""
Used to search directly in LDAP, by using ldap filters and return
fields. When req_ldap is None, returns all the entries in the LDAP.
:param req_ldap: ldap style request, with appropriate filters,
example: (cn=*).
:param expected_fields: Fields in the user ldap entry that has to be
returned. If None is provided, will return 'mail', 'givenName',
'sn', 'uid', 'sshPublicKey', 'shadowExpire'.
:type req_ldap: string
:type expected_fields: list
.. seealso:: make_ldap_filters_from_record for req_ldap format.
"""
result = self.conn.connect(bind=False)
if (result['bool']):
return_fields_list = []
if expected_fields is None:
return_fields_list = ['mail', 'givenName', 'sn', 'uid',
'sshPublicKey', 'shadowExpire']
else:
return_fields_list = expected_fields
#No specifc request specified, get the whole LDAP
if req_ldap is None:
req_ldap = '(cn=*)'
logger.debug("LDAP.PY \t LdapSearch req_ldap %s \
return_fields_list %s" \
%(req_ldap, return_fields_list))
try:
msg_id = self.conn.ldapserv.search(
self.baseDN, ldap.SCOPE_SUBTREE,
req_ldap, return_fields_list)
#Get all the results matching the search from ldap in one
#shot (1 value)
result_type, result_data = \
self.conn.ldapserv.result(msg_id, 1)
self.conn.close()
logger.debug("LDAP.PY \t LdapSearch result_data %s"
% (result_data))
return result_data
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapSearch Error %s" % error)
return []
else:
logger.error("LDAP.PY \t Connection Failed")
return
def load(self):
try:
infile = file(self.filename, 'r')
self.url2version = pickle.load(infile)
infile.close()
except:
logger.debug("Cannot load version cache, restarting from scratch")
self.url2version = {}
logger.debug("loaded version cache with %d entries %s" %
(len(self.url2version), self.url2version.keys()))
def POSTRequestToOARRestAPI(self, request, datadict, username=None):
""" Used to post a job on OAR , along with data associated
with the job.
"""
#first check that all params for are OK
try:
self.oarserver['uri'] = \
self.OAR_REQUEST_POST_URI_DICT[request]['uri']
except KeyError:
logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \
valid")
return
if datadict and 'strval' in datadict:
self.oarserver['uri'] = self.oarserver['uri'].replace("id", \
str(datadict['strval']))
del datadict['strval']
data = json.dumps(datadict)
headers = {'X-REMOTE_IDENT':username, \
'content-type': self.POST_FORMAT['json']['content'], \
'content-length':str(len(data))}
try:
conn = HTTPConnection(self.oarserver['ip'], \
self.oarserver['port'])
conn.request("POST", self.oarserver['uri'], data, headers)
resp = conn.getresponse()
body = resp.read()
except NotConnected:
logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \
data %s \r\n \t\n \t\t headers %s uri %s" \
%(data,headers,self.oarserver['uri']))
except Exception as error:
logger.log_exc("POST_OAR_SERVER : Connection error: %s " % (error))
raise Exception("POST_OAR_SERVER : Connection error %s " % (error))
finally:
conn.close()
if resp.status >= 400:
raise ValueError("Response Error %s, %s" %
(resp.status, resp.reason))
try:
answer = json.loads(body)
logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer))
return answer
except ValueError, error:
logger.log_exc("Failed to parse Server Response: error %s \
%s" % (error))
def determine_user_rights(self, caller_hrn, reg_record):
"""
Given a user credential and a record, determine what set of rights the
user should have to that record.
This is intended to replace determine_user_rights() and
verify_cancreate_credential()
"""
rl = Rights()
type = reg_record.type
logger.debug(
"entering determine_user_rights with record %s and caller_hrn %s" %
(reg_record, caller_hrn))
if type == 'slice':
# researchers in the slice are in the DB as-is
researcher_hrns = [user.hrn for user in reg_record.reg_researchers]
# locating PIs attached to that slice
slice_pis = reg_record.get_pis()
pi_hrns = [user.hrn for user in slice_pis]
if (caller_hrn in researcher_hrns + pi_hrns):
rl.add('refresh')
rl.add('embed')
rl.add('bind')
rl.add('control')
rl.add('info')
elif type == 'authority':
pi_hrns = [user.hrn for user in reg_record.reg_pis]
if (caller_hrn == self.config.SFA_INTERFACE_HRN):
rl.add('authority')
rl.add('sa')
rl.add('ma')
if (caller_hrn in pi_hrns):
rl.add('authority')
rl.add('sa')
# NOTE: for the PL implementation, this 'operators' list
# amounted to users with 'tech' role in that site
# it seems like this is not needed any longer, so for now I just drop that
# operator_hrns = reg_record.get('operator',[])
# if (caller_hrn in operator_hrns):
# rl.add('authority')
# rl.add('ma')
elif type == 'user':
rl.add('refresh')
rl.add('resolve')
rl.add('info')
elif type == 'node':
rl.add('operator')
return rl
def _create_job_structure_request_for_OAR(lease_dict):
""" Creates the structure needed for a correct POST on OAR.
Makes the timestamp transformation into the appropriate format.
Sends the POST request to create the job with the resources in
added_nodes.
"""
nodeid_list = []
reqdict = {}
reqdict['workdir'] = '/tmp'
reqdict['resource'] = "{network_address in ("
for node in lease_dict['added_nodes']:
logger.debug("\r\n \r\n OARrestapi \t \
__create_job_structure_request_for_OAR node %s" % (node))
# Get the ID of the node
nodeid = node
reqdict['resource'] += "'" + nodeid + "', "
nodeid_list.append(nodeid)
custom_length = len(reqdict['resource']) - 2
reqdict['resource'] = reqdict['resource'][0:custom_length] + \
")}/nodes=" + str(len(nodeid_list))
walltime, sleep_walltime = \
IotlabShell._process_walltime(\
int(lease_dict['lease_duration']))
reqdict['resource'] += ",walltime=" + str(walltime[0]) + \
":" + str(walltime[1]) + ":" + str(walltime[2])
reqdict['script_path'] = "/bin/sleep " + str(sleep_walltime)
#In case of a scheduled experiment (not immediate)
#To run an XP immediately, don't specify date and time in RSpec
#They will be set to None.
if lease_dict['lease_start_time'] is not '0':
#Readable time accepted by OAR
start_time = datetime.fromtimestamp( \
int(lease_dict['lease_start_time'])).\
strftime(lease_dict['time_format'])
reqdict['reservation'] = start_time
#If there is not start time, Immediate XP. No need to add special
# OAR parameters
reqdict['type'] = "deploy"
reqdict['directory'] = ""
reqdict['name'] = "SFA_" + lease_dict['slice_user']
return reqdict
def __init__(self, config=None):
if not config:
config = Config()
if has_nova:
# instantiate managers
self.auth_manager = KeystoneClient(config=config)
self.image_manager = GlanceClient(config=config)
self.nova_manager = NovaClient(config=config)
else:
logger.debug('nova access - REST')
raise SfaNotImplemented('nova access - Rest')
def func(*args, **kwds):
if name not in FdShell.direct_calls:
raise Exception, "Illegal method call %s for FEDERICA driver" % (
name)
logger.info("Issuing %s args=%s kwds=%s to federica"%\
(name,args,kwds))
# result=getattr(self.proxy, "AggregateManager.%s"%name)(credential, *args, **kwds)
result = getattr(self.proxy, "AggregateManager.%s" % name)(*args,
**kwds)
logger.debug('FdShell %s (%s) returned ... ' % (name, name))
return result
def _dispatch(self, method, params):
logger.debug("SecureXMLRPCServer._dispatch, method=%s" % method)
try:
return SimpleXMLRPCServer.SimpleXMLRPCDispatcher._dispatch(
self, method, params)
except:
# can't use format_exc() as it is not available in jython yet
# (even in trunk).
type, value, tb = sys.exc_info()
raise xmlrpclib.Fault(
1, ''.join(traceback.format_exception(type, value, tb)))
def func(*args, **kwds):
actual_name = None
if name in NitosShell.direct_calls: actual_name = name
if not actual_name:
raise Exception, "Illegal method call %s for NITOS driver" % (
name)
actual_name = "scheduler.server." + actual_name
result = getattr(self.proxy, actual_name)(*args, **kwds)
logger.debug('NitosShell %s (%s) returned ... ' %
(name, actual_name))
return result
def createtable(self):
"""
Creates all the table sof the engine.
Uses the global dictionnary holding the tablenames and the table schema.
"""
logger.debug("IOTLABPOSTGRES createtable \
IotlabBase.metadata.sorted_tables %s \r\n engine %s" %
(IotlabBase.metadata.sorted_tables, self.iotlab_engine))
IotlabBase.metadata.create_all(self.iotlab_engine)
return
def LdapGenerateUniqueLogin(self, record):
"""
Generate login for adding a new user in LDAP Directory
(four characters minimum length). Get proper last name and
first name so that the user's login can be generated.
:param record: Record must contain first_name and last_name.
:type record: dict
:returns: the generated login for the user described with record if the
login generation is successful, None if it fails.
:rtype: string or None
"""
#For compatibility with other ldap func
if 'mail' in record and 'email' not in record:
record['email'] = record['mail']
lower_first_name, lower_last_name = \
self.login_pwd.get_user_firstname_lastname(record)
index, login = self.login_pwd.choose_sets_chars_for_login(
lower_first_name, lower_last_name)
login_filter = '(uid=' + login + ')'
get_attrs = ['uid']
try:
#Check if login already in use
while (len(self.LdapSearch(login_filter, get_attrs)) is not 0):
index += 1
if index >= 9:
logger.error("LoginException : Generation login error \
with minimum four characters")
else:
try:
login = \
lower_first_name[0:index] + \
lower_last_name[0:
self.login_pwd.login_max_length
- index]
login_filter = '(uid=' + login + ')'
except KeyError:
print "lower_first_name - lower_last_name too short"
logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"
% (login))
return login
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error))
return None
