def save (self):
try:
outfile=file(self.filename,'w')
pickle.dump(self.url2version,outfile)
outfile.close()
except:
logger.log_exc ("Cannot save version cache into %s"%self.filename)
def do_POST(self):
"""Handles the HTTPS POST request.
It was copied out from SimpleXMLRPCServer.py and modified to shutdown
the socket cleanly.
"""
try:
peer_cert = Certificate()
peer_cert.load_from_pyopenssl_x509(self.connection.get_peer_certificate())
generic=Generic.the_flavour()
self.api = generic.make_api (peer_cert = peer_cert,
interface = self.server.interface,
key_file = self.server.key_file,
cert_file = self.server.cert_file,
cache = self.cache)
#logger.info("SecureXMLRpcRequestHandler.do_POST:")
#logger.info("interface=%s"%self.server.interface)
#logger.info("key_file=%s"%self.server.key_file)
#logger.info("api=%s"%self.api)
#logger.info("server=%s"%self.server)
#logger.info("handler=%s"%self)
# get arguments
request = self.rfile.read(int(self.headers["content-length"]))
remote_addr = (remote_ip, remote_port) = self.connection.getpeername()
self.api.remote_addr = remote_addr
response = self.api.handle(remote_addr, request, self.server.method_map)
except Exception, fault:
# This should only happen if the module is buggy
# internal error, report as HTTP server error
logger.log_exc("server.do_POST")
response = self.api.prepare_response(fault)
def delete_security_group(self, name):
try:
security_group = self.client.security_groups.find(name=name)
if not security_group.name == 'default':
self.client.security_groups.delete(security_group.id)
except Exception, ex:
logger.log_exc("Failed to delete security group")
def save(self):
try:
outfile = file(self.filename, 'w')
pickle.dump(self.url2version, outfile)
outfile.close()
except:
logger.log_exc("Cannot save version cache into %s" % self.filename)
def get_slice_and_slivers(self, slice_xrn, login=None):
"""
Returns a dict of slivers keyed on the sliver's node_id
"""
slivers = {}
sfa_slice = None
if not slice_xrn:
return (sfa_slice, slivers)
slice_urn = hrn_to_urn(slice_xrn, 'slice')
slice_hrn, _ = urn_to_hrn(slice_xrn)
slice_name = slice_hrn
slices = self.driver.GetSlices(slice_filter= str(slice_name), \
slice_filter_type = 'slice_hrn', login=login)
logger.debug("Slabaggregate api \tget_slice_and_slivers \
sfa_slice %s \r\n slices %s self.driver.hrn %s" \
%(sfa_slice, slices, self.driver.hrn))
if not slices:
return (sfa_slice, slivers)
#if isinstance(sfa_slice, list):
#sfa_slice = slices[0]
#else:
#sfa_slice = slices
# sort slivers by node id , if there is a job
#and therfore, node allocated to this slice
for sfa_slice in slices:
try:
node_ids_list = sfa_slice['node_ids']
except KeyError:
logger.log_exc("SLABAGGREGATE \t \
get_slice_and_slivers KeyError ")
continue
for node in node_ids_list:
sliver_xrn = Xrn(slice_urn, type='sliver', id=node)
sliver_xrn.set_authority(self.driver.hrn)
#node_id = self.driver.root_auth + '.' + node_id
sliver = Sliver({'sliver_id':sliver_xrn.urn,
'name': sfa_slice['hrn'],
'type': 'slab-node',
'tags': []})
slivers[node] = sliver
#Add default sliver attribute :
#connection information for senslab
if get_authority (sfa_slice['hrn']) == self.driver.root_auth:
tmp = sfa_slice['hrn'].split('.')
ldap_username = tmp[1].split('_')[0]
vmaddr = 'ssh ' + ldap_username + '@grenoble.senslab.info'
slivers['default_sliver'] = {'vm': vmaddr , 'login': ldap_username}
#TODO get_slice_and_slivers Find the login of the external user
logger.debug("SLABAGGREGATE api get_slice_and_slivers slivers %s "\
%(slivers))
return (slices, slivers)
def convert_public_key(key):
keyconvert_path = "/usr/bin/keyconvert.py"
if not os.path.isfile(keyconvert_path):
raise IOError, "Could not find keyconvert in %s" % keyconvert_path
# we can only convert rsa keys
if "ssh-dss" in key:
return None
(ssh_f, ssh_fn) = tempfile.mkstemp()
ssl_fn = tempfile.mktemp()
os.write(ssh_f, key)
os.close(ssh_f)
cmd = keyconvert_path + " " + ssh_fn + " " + ssl_fn
os.system(cmd)
# this check leaves the temporary file containing the public key so
# that it can be expected to see why it failed.
# TODO: for production, cleanup the temporary files
if not os.path.exists(ssl_fn):
return None
k = Keypair()
try:
k.load_pubkey_from_file(ssl_fn)
except:
logger.log_exc("convert_public_key caught exception")
k = None
# remove the temporary files
os.remove(ssh_fn)
os.remove(ssl_fn)
return k
def nuke (self):
model.drop_tables(self.engine)
# so in this case it's like we haven't initialized the db at all
try:
migrate.drop_version_control (self.url, self.repository)
except migrate.exceptions.DatabaseNotControlledError:
logger.log_exc("Failed to drop version control")
def convert_public_key(key):
keyconvert_path = "/usr/bin/keyconvert.py"
if not os.path.isfile(keyconvert_path):
raise IOError, "Could not find keyconvert in %s" % keyconvert_path
# we can only convert rsa keys
if "ssh-dss" in key:
return None
(ssh_f, ssh_fn) = tempfile.mkstemp()
ssl_fn = tempfile.mktemp()
os.write(ssh_f, key)
os.close(ssh_f)
cmd = keyconvert_path + " " + ssh_fn + " " + ssl_fn
os.system(cmd)
# this check leaves the temporary file containing the public key so
# that it can be expected to see why it failed.
# TODO: for production, cleanup the temporary files
if not os.path.exists(ssl_fn):
return None
k = Keypair()
try:
k.load_pubkey_from_file(ssl_fn)
except:
logger.log_exc("convert_public_key caught exception")
k = None
# remove the temporary files
os.remove(ssh_fn)
os.remove(ssl_fn)
return k
def do_POST(self):
"""Handles the HTTPS POST request.
It was copied out from SimpleXMLRPCServer.py and modified to shutdown
the socket cleanly.
"""
try:
peer_cert = Certificate()
peer_cert.load_from_pyopenssl_x509(
self.connection.get_peer_certificate())
generic = Generic.the_flavour()
self.api = generic.make_api(peer_cert=peer_cert,
interface=self.server.interface,
key_file=self.server.key_file,
cert_file=self.server.cert_file,
cache=self.cache)
#logger.info("SecureXMLRpcRequestHandler.do_POST:")
#logger.info("interface=%s"%self.server.interface)
#logger.info("key_file=%s"%self.server.key_file)
#logger.info("api=%s"%self.api)
#logger.info("server=%s"%self.server)
#logger.info("handler=%s"%self)
# get arguments
request = self.rfile.read(int(self.headers["content-length"]))
remote_addr = (remote_ip,
remote_port) = self.connection.getpeername()
self.api.remote_addr = remote_addr
response = self.api.handle(remote_addr, request,
self.server.method_map)
except Exception, fault:
# This should only happen if the module is buggy
# internal error, report as HTTP server error
logger.log_exc("server.do_POST")
response = self.api.prepare_response(fault)
def GETRequestToOARRestAPI(self, request, strval=None ,next_page=None, username = None ):
self.oarserver['uri'] = \
OARGETParser.OARrequests_uri_dict[request]['uri']
#Get job details with username
if 'owner' in OARGETParser.OARrequests_uri_dict[request] and username:
self.oarserver['uri'] += OARGETParser.OARrequests_uri_dict[request]['owner'] + username
headers = {}
data = json.dumps({})
logger.debug("OARrestapi \tGETRequestToOARRestAPI %s" %(request))
if strval:
self.oarserver['uri'] = self.oarserver['uri'].\
replace("id",str(strval))
if next_page:
self.oarserver['uri'] += next_page
if username:
headers['X-REMOTE_IDENT'] = username
print>>sys.stderr, " \r\n \t OARrestapi \tGETRequestToOARRestAPI %s" %( self.oarserver['uri'])
logger.debug("OARrestapi: \t GETRequestToOARRestAPI \
self.oarserver['uri'] %s strval %s" \
%(self.oarserver['uri'], strval))
try :
#seems that it does not work if we don't add this
headers['content-length'] = '0'
conn = HTTPConnection(self.oarserver['ip'], \
self.oarserver['port'])
conn.request("GET", self.oarserver['uri'], data, headers)
resp = ( conn.getresponse()).read()
conn.close()
except HTTPException, error :
logger.log_exc("GET_OAR_SRVR : Problem with OAR server : %s " \
%(error))
def LdapModify(self, dn, old_attributes_dict, new_attributes_dict):
""" Modifies a LDAP entry, replaces user's old attributes with
the new ones given.
:param dn: user's absolute name in the LDAP hierarchy.
:param old_attributes_dict: old user's attributes. Keys must match
the ones used in the LDAP model.
:param new_attributes_dict: new user's attributes. Keys must match
the ones used in the LDAP model.
:type dn: string
:type old_attributes_dict: dict
:type new_attributes_dict: dict
:returns: dict bool True if Successful, bool False if not.
:rtype: dict
"""
ldif = modlist.modifyModlist(old_attributes_dict, new_attributes_dict)
# Connect and bind/authenticate
result = self.conn.connect()
if (result['bool']):
try:
self.conn.ldapserv.modify_s(dn, ldif)
self.conn.close()
return {'bool': True}
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapModify Error %s" % error)
return {'bool': False}
def ParseJobsIds(self):
job_resources = ['wanted_resources', 'name', 'id', 'start_time', \
'state','owner','walltime','message']
job_resources_full = ['launching_directory', 'links', \
'resubmit_job_id', 'owner', 'events', 'message', \
'scheduled_start', 'id', 'array_id', 'exit_code', \
'properties', 'state','array_index', 'walltime', \
'type', 'initial_request', 'stop_time', 'project',\
'start_time', 'dependencies','api_timestamp','submission_time', \
'reservation', 'stdout_file', 'types', 'cpuset_name', \
'name', 'wanted_resources','queue','stderr_file','command']
job_info = self.raw_json
#logger.debug("OARESTAPI ParseJobsIds %s" %(self.raw_json))
values = []
try:
for k in job_resources:
values.append(job_info[k])
return dict(zip(job_resources, values))
except KeyError:
logger.log_exc("ParseJobsIds KeyError ")
def remove_rule_from_group(self,
group_name=None,
protocol='tcp',
cidr_ip='0.0.0.0/0',
port_range=None,
icmp_type_code=None,
source_group_name=None,
source_group_owner_id=None):
try:
from_port, to_port = self._validate_port_range(port_range)
icmp_type = self._validate_icmp_type_code(icmp_type_code)
if icmp_type:
from_port, to_port = icmp_type[0], icmp_type[1]
group = self.client.security_groups.find(name=group_name)
filter = {
'id': group.id,
'from_port': from_port,
'to_port': to_port,
'cidr_ip': ip,
'ip_protocol': protocol,
}
rule = self.client.security_group_rules.find(**filter)
if rule:
self.client.security_group_rules.delete(rule)
except Exception, ex:
logger.log_exc("Failed to remove rule from group %s" % group_name)
def nuke(self):
model.drop_tables(self.engine)
# so in this case it's like we haven't initialized the db at all
try:
migrate.drop_version_control(self.url, self.repository)
except migrate.exceptions.DatabaseNotControlledError:
logger.log_exc("Failed to drop version control")
def _process_walltime(duration):
""" Calculates the walltime in seconds from the duration in H:M:S
specified in the RSpec.
"""
if duration:
# Fixing the walltime by adding a few delays.
# First put the walltime in seconds oarAdditionalDelay = 20;
# additional delay for /bin/sleep command to
# take in account prologue and epilogue scripts execution
# int walltimeAdditionalDelay = 240; additional delay
#for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
#in oar.conf
# Put the duration in seconds first
#desired_walltime = duration * 60
desired_walltime = duration
total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
walltime = []
#Put the walltime back in str form
#First get the hours
walltime.append(str(total_walltime / 3600))
total_walltime = total_walltime - 3600 * int(walltime[0])
#Get the remaining minutes
walltime.append(str(total_walltime / 60))
total_walltime = total_walltime - 60 * int(walltime[1])
#Get the seconds
walltime.append(str(total_walltime))
else:
logger.log_exc(" __process_walltime duration null")
return walltime, sleep_walltime
def _process_walltime(duration):
""" Calculates the walltime in seconds from the duration in H:M:S
specified in the RSpec.
"""
if duration:
# Fixing the walltime by adding a few delays.
# First put the walltime in seconds oarAdditionalDelay = 20;
# additional delay for /bin/sleep command to
# take in account prologue and epilogue scripts execution
# int walltimeAdditionalDelay = 240; additional delay
#for prologue/epilogue execution = $SERVER_PROLOGUE_EPILOGUE_TIMEOUT
#in oar.conf
# Put the duration in seconds first
#desired_walltime = duration * 60
desired_walltime = duration
total_walltime = desired_walltime + 240 #+4 min Update SA 23/10/12
sleep_walltime = desired_walltime # 0 sec added Update SA 23/10/12
walltime = []
#Put the walltime back in str form
#First get the hours
walltime.append(str(total_walltime / 3600))
total_walltime = total_walltime - 3600 * int(walltime[0])
#Get the remaining minutes
walltime.append(str(total_walltime / 60))
total_walltime = total_walltime - 60 * int(walltime[1])
#Get the seconds
walltime.append(str(total_walltime))
else:
logger.log_exc(" __process_walltime duration null")
return walltime, sleep_walltime
def verify_slice_nodes(self, slice_urn, slice, rspec_nodes):
slivers = {}
for node in rspec_nodes:
hostname = node.get('component_name')
client_id = node.get('client_id')
#client_id = node.get('component_id')
component_id = node.get('component_id').strip()
if hostname:
hostname = hostname.strip()
elif component_id:
hostname = xrn_to_hostname(component_id)
if hostname:
slivers[hostname] = {'client_id': client_id, 'component_id': component_id}
all_nodes = self.driver.shell.GetNodes()
requested_slivers = []
for node in all_nodes:
if node['hostname'] in slivers.keys():
requested_slivers.append(node['node_id'])
if 'node_ids' not in slice.keys():
slice['node_ids']=[]
nodes = self.driver.shell.GetNodes({'node_ids': slice['node_ids']})
current_slivers = [node['node_id'] for node in nodes]
# remove nodes not in rspec
deleted_nodes = list(set(current_slivers).difference(requested_slivers))
# add nodes from rspec
added_nodes = list(set(requested_slivers).difference(current_slivers))
try:
self.driver.shell.AddSliceToNodes({'slice_id': slice['slice_id'], 'node_ids': added_nodes})
self.driver.shell.DeleteSliceFromNodes({'slice_id': slice['slice_id'], 'node_ids': deleted_nodes})
except:
logger.log_exc('Failed to add/remove slice from nodes')
slices = self.driver.shell.GetSlices({'slice_name': slice['slice_name']})
resulting_nodes = self.driver.shell.GetNodes({'node_ids': slices[0]['node_ids']})
# update sliver allocations
for node in resulting_nodes:
client_id = slivers[node['hostname']]['client_id']
component_id = slivers[node['hostname']]['component_id']
sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'], node['node_id'])
sliver_id = Xrn(sliver_hrn, type='sliver').urn
record = SliverAllocation(sliver_id=sliver_id, client_id=client_id,
component_id=component_id,
slice_urn = slice_urn,
allocation_state='geni_allocated')
record.sync(self.driver.api.dbsession())
return resulting_nodes
def merge_rspecs(rspecs):
"""
Merge merge a list of RSpecs into 1 RSpec, and return the result.
rspecs must be a valid RSpec string or list of RSpec strings.
"""
if not rspecs or not isinstance(rspecs, list):
return rspecs
# ugly hack to avoid sending the same info twice, when the call graph has dags
known_networks={}
def register_network (network):
try:
known_networks[network.get('name')]=True
except:
logger.error("merge_rspecs: cannot register network with no name in rspec")
pass
def is_registered_network (network):
try:
return network.get('name') in known_networks
except:
logger.error("merge_rspecs: cannot retrieve network with no name in rspec")
return False
# the resulting tree
rspec = None
for input_rspec in rspecs:
# ignore empty strings as returned with used call_ids
if not input_rspec: continue
try:
tree = etree.parse(StringIO(input_rspec))
except etree.XMLSyntaxError:
# consider failing silently here
logger.log_exc("merge_rspecs, parse error")
message = str(sys.exc_info()[1]) + ' with ' + input_rspec
raise InvalidRSpec(message)
root = tree.getroot()
if not root.get("type") in ["SFA"]:
logger.error("merge_rspecs: unexpected type for rspec root, %s"%root.get('type'))
continue
if rspec == None:
# we scan the first input, register all networks
# in addition we remove duplicates - needed until everyone runs 1.0-10
rspec = root
for network in root.iterfind("./network"):
if not is_registered_network(network):
register_network(network)
else:
# duplicate in the first input - trash it
root.remove(network)
else:
for network in root.iterfind("./network"):
if not is_registered_network(network):
rspec.append(deepcopy(network))
register_network(network)
for request in root.iterfind("./request"):
rspec.append(deepcopy(request))
return etree.tostring(rspec, xml_declaration=True, pretty_print=True)
def run_instances(self, instance_name, tenant_name, rspec, key_name, pubkeys):
#logger.debug('Reserving an instance: image: %s, flavor: ' \
# '%s, key: %s, name: %s' % \
# (image_id, flavor_id, key_name, slicename))
# make sure a tenant exists for this slice
tenant = self.create_tenant(tenant_name)
# add the sfa admin user to this tenant and update our nova client connection
# to use these credentials for the rest of this session. This emsures that the instances
# we create will be assigned to the correct tenant.
sfa_admin_user = self.driver.shell.auth_manager.users.find(name=self.driver.shell.auth_manager.opts['OS_USERNAME'])
user_role = self.driver.shell.auth_manager.roles.find(name='user')
admin_role = self.driver.shell.auth_manager.roles.find(name='admin')
self.driver.shell.auth_manager.roles.add_user_role(sfa_admin_user, admin_role, tenant)
self.driver.shell.auth_manager.roles.add_user_role(sfa_admin_user, user_role, tenant)
self.driver.shell.nova_manager.connect(tenant=tenant.name)
authorized_keys = "\n".join(pubkeys)
files = {'/root/.ssh/authorized_keys': authorized_keys}
rspec = RSpec(rspec)
requested_instances = defaultdict(list)
# iterate over clouds/zones/nodes
created_instances = []
for node in rspec.version.get_nodes_with_slivers():
instances = node.get('slivers', [])
if not instances:
continue
for instance in instances:
try:
metadata = {}
flavor_id = self.driver.shell.nova_manager.flavors.find(name=instance['name'])
image = instance.get('disk_image')
if image and isinstance(image, list):
image = image[0]
else:
raise InvalidRSpec("Must specify a disk_image for each VM")
image_id = self.driver.shell.nova_manager.images.find(name=image['name'])
fw_rules = instance.get('fw_rules', [])
group_name = self.create_security_group(instance_name, fw_rules)
metadata['security_groups'] = group_name
if node.get('component_id'):
metadata['component_id'] = node['component_id']
if node.get('client_id'):
metadata['client_id'] = node['client_id']
server = self.driver.shell.nova_manager.servers.create(flavor=flavor_id,
image=image_id,
key_name = key_name,
security_groups = [group_name],
files=files,
meta=metadata,
name=instance_name)
created_instances.append(server)
except Exception, err:
logger.log_exc(err)
def run_instances(self, instance_name, tenant_name, rspec, key_name, pubkeys):
#logger.debug('Reserving an instance: image: %s, flavor: ' \
# '%s, key: %s, name: %s' % \
# (image_id, flavor_id, key_name, slicename))
# make sure a tenant exists for this slice
tenant = self.create_tenant(tenant_name)
# add the sfa admin user to this tenant and update our nova client connection
# to use these credentials for the rest of this session. This emsures that the instances
# we create will be assigned to the correct tenant.
sfa_admin_user = self.driver.shell.auth_manager.users.find(name=self.driver.shell.auth_manager.opts['OS_USERNAME'])
user_role = self.driver.shell.auth_manager.roles.find(name='user')
admin_role = self.driver.shell.auth_manager.roles.find(name='admin')
self.driver.shell.auth_manager.roles.add_user_role(sfa_admin_user, admin_role, tenant)
self.driver.shell.auth_manager.roles.add_user_role(sfa_admin_user, user_role, tenant)
self.driver.shell.nova_manager.connect(tenant=tenant.name)
authorized_keys = "\n".join(pubkeys)
files = {'/root/.ssh/authorized_keys': authorized_keys}
rspec = RSpec(rspec)
requested_instances = defaultdict(list)
# iterate over clouds/zones/nodes
slivers = []
for node in rspec.version.get_nodes_with_slivers():
instances = node.get('slivers', [])
if not instances:
continue
for instance in instances:
try:
metadata = {}
flavor_id = self.driver.shell.nova_manager.flavors.find(name=instance['name'])
image = instance.get('disk_image')
if image and isinstance(image, list):
image = image[0]
else:
raise InvalidRSpec("Must specify a disk_image for each VM")
image_id = self.driver.shell.nova_manager.images.find(name=image['name'])
fw_rules = instance.get('fw_rules', [])
group_name = self.create_security_group(instance_name, fw_rules)
metadata['security_groups'] = group_name
if node.get('component_id'):
metadata['component_id'] = node['component_id']
if node.get('client_id'):
metadata['client_id'] = node['client_id']
server = self.driver.shell.nova_manager.servers.create(
flavor=flavor_id,
image=image_id,
key_name = key_name,
security_groups = [group_name],
files=files,
meta=metadata,
name=instance_name)
slivers.append(server)
except Exception, err:
logger.log_exc(err)
def LdapSearch(self, req_ldap=None, expected_fields=None):
"""
Used to search directly in LDAP, by using ldap filters and return
fields. When req_ldap is None, returns all the entries in the LDAP.
:param req_ldap: ldap style request, with appropriate filters,
example: (cn=*).
:param expected_fields: Fields in the user ldap entry that has to be
returned. If None is provided, will return 'mail', 'givenName',
'sn', 'uid', 'sshPublicKey', 'shadowExpire'.
:type req_ldap: string
:type expected_fields: list
.. seealso:: make_ldap_filters_from_record for req_ldap format.
"""
result = self.conn.connect(bind=False)
if (result['bool']):
return_fields_list = []
if expected_fields is None:
return_fields_list = ['mail', 'givenName', 'sn', 'uid',
'sshPublicKey', 'shadowExpire']
else:
return_fields_list = expected_fields
#No specifc request specified, get the whole LDAP
if req_ldap is None:
req_ldap = '(cn=*)'
logger.debug("LDAP.PY \t LdapSearch req_ldap %s \
return_fields_list %s" \
%(req_ldap, return_fields_list))
try:
msg_id = self.conn.ldapserv.search(
self.baseDN, ldap.SCOPE_SUBTREE,
req_ldap, return_fields_list)
#Get all the results matching the search from ldap in one
#shot (1 value)
result_type, result_data = \
self.conn.ldapserv.result(msg_id, 1)
self.conn.close()
logger.debug("LDAP.PY \t LdapSearch result_data %s"
% (result_data))
return result_data
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapSearch Error %s" % error)
return []
else:
logger.error("LDAP.PY \t Connection Failed")
return
def _Provision(aggregate, server, xrn, credential, options):
tStart = time.time()
try:
# Need to call GetVersion at an aggregate to determine the supported
# rspec type/format beofre calling CreateSliver at an Aggregate.
server_version = api.get_cached_server_version(server)
result = server.Provision(xrn, credential, options)
return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
except:
logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
lease_start_time, lease_duration, slice_user=None):
"""
Create a job request structure based on the information provided
and post the job on OAR.
:param added_nodes: list of nodes that belong to the described lease.
:param slice_name: the slice hrn associated to the lease.
:param lease_start_time: timestamp of the lease startting time.
:param lease_duration: lease durationin minutes
"""
lease_dict = {}
lease_dict['lease_start_time'] = lease_start_time
lease_dict['lease_duration'] = lease_duration
lease_dict['added_nodes'] = added_nodes
lease_dict['slice_name'] = slice_name
lease_dict['slice_user'] = slice_user
lease_dict['grain'] = self.GetLeaseGranularity()
# I don't know why the SFATIME_FORMAT has changed...
# from sfa.util.sfatime import SFATIME_FORMAT
# Let's use a fixed format %Y-%m-%d %H:%M:%S
#lease_dict['time_format'] = self.time_format
lease_dict['time_format'] = '%Y-%m-%d %H:%M:%S'
logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\
\r\n " %(slice_user))
#Create the request for OAR
reqdict = self._create_job_structure_request_for_OAR(lease_dict)
# first step : start the OAR job and update the job
logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\
\r\n " %(reqdict))
answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
reqdict, slice_user)
logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s " %(answer))
try:
jobid = answer['id']
except KeyError:
logger.log_exc("IOTLAB_API \tLaunchExperimentOnOAR \
Impossible to create job %s " %(answer))
return None
if jobid :
logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s \
added_nodes %s slice_user %s" %(jobid, added_nodes, \
slice_user))
return jobid
def verify_slice_leases(self, slice, rspec_requested_leases, peer):
leases = self.driver.shell.GetLeases({'name':slice['name'], 'clip':int(time.time())}, ['lease_id','name', 'hostname', 't_from', 't_until'])
grain = self.driver.shell.GetLeaseGranularity()
requested_leases = []
for lease in rspec_requested_leases:
requested_lease = {}
slice_name = hrn_to_pl_slicename(lease['slice_id'])
if slice_name != slice['name']:
continue
elif Xrn(lease['component_id']).get_authority_urn().split(':')[0] != self.driver.hrn:
continue
hostname = xrn_to_hostname(lease['component_id'])
# fill the requested node with nitos ids
requested_lease['name'] = slice['name']
requested_lease['hostname'] = hostname
requested_lease['t_from'] = int(lease['start_time'])
requested_lease['t_until'] = int(lease['duration']) * grain + int(lease['start_time'])
requested_leases.append(requested_lease)
# prepare actual slice leases by lease_id
leases_by_id = {}
for lease in leases:
leases_by_id[lease['lease_id']] = {'name': lease['name'], 'hostname': lease['hostname'], \
't_from': lease['t_from'], 't_until': lease['t_until']}
added_leases = []
kept_leases_id = []
deleted_leases_id = []
for lease_id in leases_by_id:
if leases_by_id[lease_id] not in requested_leases:
deleted_leases_id.append(lease_id)
else:
kept_leases_id.append(lease_id)
requested_leases.remove(leases_by_id[lease_id])
added_leases = requested_leases
try:
if peer:
self.driver.shell.UnBindObjectFromPeer('slice', slice['slice_id'], peer['shortname'])
self.driver.shell.DeleteLeases(deleted_leases_id)
for lease in added_leases:
self.driver.shell.AddLeases(lease['hostname'], slice['name'], lease['t_from'], lease['t_until'])
except:
logger.log_exc('Failed to add/remove slice leases')
return leases
def _Renew(aggregate, server, xrn, creds, expiration_time, options):
try:
result=server.Renew(xrn, creds, expiration_time, options)
if type(result)!=dict:
result = {'code': {'geni_code': 0}, 'value': result}
result['aggregate'] = aggregate
return result
except:
logger.log_exc('Something wrong in _Renew with URL %s'%server.url)
return {'aggregate': aggregate, 'exc_info': traceback.format_exc(),
'code': {'geni_code': -1},
'value': False, 'output': ""}
def LdapGenerateUniqueLogin(self, record):
"""
Generate login for adding a new user in LDAP Directory
(four characters minimum length). Get proper last name and
first name so that the user's login can be generated.
:param record: Record must contain first_name and last_name.
:type record: dict
:returns: the generated login for the user described with record if the
login generation is successful, None if it fails.
:rtype: string or None
"""
#For compatibility with other ldap func
if 'mail' in record and 'email' not in record:
record['email'] = record['mail']
lower_first_name, lower_last_name = \
self.login_pwd.get_user_firstname_lastname(record)
index, login = self.login_pwd.choose_sets_chars_for_login(
lower_first_name, lower_last_name)
login_filter = '(uid=' + login + ')'
get_attrs = ['uid']
try:
#Check if login already in use
while (len(self.LdapSearch(login_filter, get_attrs)) is not 0):
index += 1
if index >= 9:
logger.error("LoginException : Generation login error \
with minimum four characters")
else:
try:
login = \
lower_first_name[0:index] + \
lower_last_name[0:
self.login_pwd.login_max_length
- index]
login_filter = '(uid=' + login + ')'
except KeyError:
print "lower_first_name - lower_last_name too short"
logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"
% (login))
return login
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error))
return None
def verify_slice_leases_nodes(self, slice, rspec_requested_nodes):
nodes = self.driver.shell.getNodes({}, [])
requested_nodes = []
for node in rspec_requested_nodes:
requested_node = {}
nitos_nodes = []
nitos_nodes.extend(nodes)
slice_name = hrn_to_nitos_slicename(node['slice_id'])
if slice_name != slice['slice_name']:
continue
hostname = xrn_to_hostname(node['component_id'])
nitos_node = self.driver.filter_nitos_results(nitos_nodes, {'hostname': hostname})
if not nitos_node:
continue
nitos_node = nitos_node[0]
# fill the requested node with nitos ids
requested_node['slice_id'] = slice['slice_id']
requested_node['node_id'] = nitos_node['node_id']
requested_node['start_time'] = node['start_time']
requested_node['end_time'] = str(int(node['duration']) * int(self.driver.testbedInfo['grain']) + int(node['start_time']))
requested_nodes.append(requested_node)
# get actual nodes reservation data for the slice
reserved_nodes = self.driver.filter_nitos_results(self.driver.shell.getReservedNodes({}, []), {'slice_id': slice['slice_id']})
reserved_nodes_by_id = {}
for node in reserved_nodes:
reserved_nodes_by_id[node['reservation_id']] = {'slice_id': node['slice_id'], \
'node_id': node['node_id'], 'start_time': node['start_time'], \
'end_time': node['end_time']}
added_nodes = []
kept_nodes_id = []
deleted_nodes_id = []
for reservation_id in reserved_nodes_by_id:
if reserved_nodes_by_id[reservation_id] not in requested_nodes:
deleted_nodes_id.append(reservation_id)
else:
kept_nodes_id.append(reservation_id)
requested_nodes.remove(reserved_nodes_by_id[reservation_id])
added_nodes = requested_nodes
try:
deleted=self.driver.shell.releaseNodes({'reservation_ids': deleted_nodes_id})
for node in added_nodes:
added=self.driver.shell.reserveNodes({'slice_id': slice['slice_id'], 'start_time': node['start_time'], 'end_time': node['end_time'], 'nodes': [node['node_id']]})
except:
logger.log_exc('Failed to add/remove slice leases nodes')
return added_nodes
def verify_slice_nodes(self, slice_urn, slice, rspec_nodes):
slivers = {}
for node in rspec_nodes:
hostname = node.get('component_name')
client_id = node.get('client_id')
component_id = node.get('component_id').strip()
if hostname:
hostname = hostname.strip()
elif component_id:
hostname = xrn_to_hostname(component_id)
if hostname:
slivers[hostname] = {
'client_id': client_id,
'component_id': component_id
}
nodes = self.driver.shell.GetNodes(
slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
current_slivers = [node['hostname'] for node in nodes]
# remove nodes not in rspec
deleted_nodes = list(set(current_slivers).difference(slivers.keys()))
# add nodes from rspec
added_nodes = list(set(slivers.keys()).difference(current_slivers))
try:
self.driver.shell.AddSliceToNodes(slice['name'], added_nodes)
self.driver.shell.DeleteSliceFromNodes(slice['name'],
deleted_nodes)
except:
logger.log_exc('Failed to add/remove slice from nodes')
slices = self.driver.shell.GetSlices(slice['name'], ['node_ids'])
resulting_nodes = self.driver.shell.GetNodes(slices[0]['node_ids'])
# update sliver allocations
for node in resulting_nodes:
client_id = slivers[node['hostname']]['client_id']
component_id = slivers[node['hostname']]['component_id']
sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'],
node['node_id'])
sliver_id = Xrn(sliver_hrn, type='sliver').urn
record = SliverAllocation(sliver_id=sliver_id,
client_id=client_id,
component_id=component_id,
slice_urn=slice_urn,
allocation_state='geni_allocated')
record.sync(self.driver.api.dbsession())
return resulting_nodes
def the_flavour (flavour=None, config=None):
if config is None: config=Config()
if flavour is None: flavour=config.SFA_GENERIC_FLAVOUR
flavour = flavour.lower()
#mixed = flavour.capitalize()
module_path="sfa.generic.%s"%flavour
classname="%s"%flavour
logger.debug("Generic.the_flavour with flavour=%s"%flavour)
try:
module = __import__ (module_path, globals(), locals(), [classname])
return getattr(module, classname)(flavour,config)
except:
logger.log_exc("Cannot locate generic instance with flavour=%s"%flavour)
def LdapAddUser(self, record) :
"""Add SFA user to LDAP if it is not in LDAP yet.
:param record: dictionnary with the user's data.
:returns: a dictionary with the status (Fail= False, Success= True)
and the uid of the newly added user if successful, or the error
message it is not. Dict has keys bool and message in case of
failure, and bool uid in case of success.
:rtype: dict
.. seealso:: make_ldap_filters_from_record
"""
logger.debug(" \r\n \t LDAP LdapAddUser \r\n\r\n ================\r\n ")
user_ldap_attrs = self.make_ldap_attributes_from_record(record)
#Check if user already in LDAP wih email, first name and last name
filter_by = self.make_ldap_filters_from_record(user_ldap_attrs)
user_exist = self.LdapSearch(filter_by)
if user_exist:
logger.warning(" \r\n \t LDAP LdapAddUser user %s %s \
already exists" % (user_ldap_attrs['sn'],
user_ldap_attrs['mail']))
return {'bool': False}
#Bind to the server
result = self.conn.connect()
if(result['bool']):
# A dict to help build the "body" of the object
logger.debug(" \r\n \t LDAP LdapAddUser attrs %s "
% user_ldap_attrs)
# The dn of our new entry/object
dn = 'uid=' + user_ldap_attrs['uid'] + "," + self.baseDN
try:
ldif = modlist.addModlist(user_ldap_attrs)
logger.debug("LDAPapi.py add attrs %s \r\n ldif %s"
% (user_ldap_attrs, ldif))
self.conn.ldapserv.add_s(dn, ldif)
logger.info("Adding user %s login %s in LDAP"
% (user_ldap_attrs['cn'], user_ldap_attrs['uid']))
except ldap.LDAPError, error:
logger.log_exc("LDAP Add Error %s" % error)
return {'bool': False, 'message': error}
self.conn.close()
return {'bool': True, 'uid': user_ldap_attrs['uid']}
def GetNodes(self, node_filter_dict=None, return_fields_list=None):
"""
Make a list of iotlab nodes and their properties from information
given by OAR. Search for specific nodes if some filters are
specified. Nodes properties returned if no return_fields_list given:
'hrn','archi','mobile','hostname','site','boot_state','node_id',
'radio','posx','posy','oar_id','posz'.
:param node_filter_dict: dictionnary of lists with node properties. For
instance, if you want to look for a specific node with its hrn,
the node_filter_dict should be {'hrn': [hrn_of_the_node]}
:type node_filter_dict: dict
:param return_fields_list: list of specific fields the user wants to be
returned.
:type return_fields_list: list
:returns: list of dictionaries with node properties
:rtype: list
"""
node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
node_dict_list = node_dict_by_id.values()
logger.debug (" IOTLAB_API GetNodes node_filter_dict %s \
return_fields_list %s " % (node_filter_dict, return_fields_list))
#No filtering needed return the list directly
if not (node_filter_dict or return_fields_list):
return node_dict_list
return_node_list = []
if node_filter_dict:
for filter_key in node_filter_dict:
try:
#Filter the node_dict_list by each value contained in the
#list node_filter_dict[filter_key]
for value in node_filter_dict[filter_key]:
for node in node_dict_list:
if node[filter_key] == value:
if return_fields_list:
tmp = {}
for k in return_fields_list:
tmp[k] = node[k]
return_node_list.append(tmp)
else:
return_node_list.append(node)
except KeyError:
logger.log_exc("GetNodes KeyError")
return
return return_node_list
def the_flavour(flavour=None, config=None):
if config is None: config = Config()
if flavour is None: flavour = config.SFA_GENERIC_FLAVOUR
flavour = flavour.lower()
#mixed = flavour.capitalize()
module_path = "sfa.generic.%s" % flavour
classname = "%s" % flavour
logger.debug("Generic.the_flavour with flavour=%s" % flavour)
try:
module = __import__(module_path, globals(), locals(), [classname])
return getattr(module, classname)(flavour, config)
except:
logger.log_exc("Cannot locate generic instance with flavour=%s" %
flavour)
def LdapModify(self, dn, old_attributes_dict, new_attributes_dict):
""" Modifies a LDAP entry """
ldif = modlist.modifyModlist(old_attributes_dict,new_attributes_dict)
# Connect and bind/authenticate
result = self.conn.connect()
if (result['bool']):
try:
self.conn.ldapserv.modify_s(dn,ldif)
self.conn.close()
return {'bool' : True }
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapModify Error %s" %error)
return {'bool' : False }
def __init__(self, manager, interface, config):
if isinstance (manager, ModuleType):
# old-fashioned module implementation
self.manager = manager
elif isinstance (manager, ClassType):
# create an instance; we don't pass the api in argument as it is passed
# to the actual method calls anyway
self.manager = manager(config)
else:
# that's what happens when there's something wrong with the db
# or any bad stuff of that kind at startup time
logger.log_exc("Failed to create a manager, startup sequence is broken")
raise SfaAPIError,"Argument to ManagerWrapper must be a module or class"
self.interface = interface
def verify_slice_leases_channels(self, slice, rspec_requested_channels):
channels = self.driver.shell.getChannels({}, [])
requested_channels = []
for channel in rspec_requested_channels:
requested_channel = {}
nitos_channels = []
nitos_channels.extend(channels)
slice_name = hrn_to_nitos_slicename(channel['slice_id'])
if slice_name != slice['slice_name']:
continue
channel_num = xrn_to_channel(channel['component_id'])
nitos_channel = self.driver.filter_nitos_results(nitos_channels, {'channel': channel_num})[0]
# fill the requested channel with nitos ids
requested_channel['slice_id'] = slice['slice_id']
requested_channel['channel_id'] = nitos_channel['channel_id']
requested_channel['start_time'] = channel['start_time']
requested_channel['end_time'] = str(int(channel['duration']) * int(self.driver.testbedInfo['grain']) + int(channel['start_time']))
requested_channels.append(requested_channel)
# get actual channel reservation data for the slice
reserved_channels = self.driver.filter_nitos_results(self.driver.shell.getReservedChannels(), {'slice_id': slice['slice_id']})
reserved_channels_by_id = {}
for channel in reserved_channels:
reserved_channels_by_id[channel['reservation_id']] = {'slice_id': channel['slice_id'], \
'channel_id': channel['channel_id'], 'start_time': channel['start_time'], \
'end_time': channel['end_time']}
added_channels = []
kept_channels_id = []
deleted_channels_id = []
for reservation_id in reserved_channels_by_id:
if reserved_channels_by_id[reservation_id] not in requested_channels:
deleted_channels_id.append(reservation_id)
else:
kept_channels_id.append(reservation_id)
requested_channels.remove(reserved_channels_by_id[reservation_id])
added_channels = requested_channels
try:
deleted=self.driver.shell.releaseChannels({'reservation_ids': deleted_channels_id})
for channel in added_channels:
added=self.driver.shell.reserveChannels({'slice_id': slice['slice_id'], 'start_time': channel['start_time'], 'end_time': channel['end_time'], 'channels': [channel['channel_id']]})
except:
logger.log_exc('Failed to add/remove slice leases channels')
return added_channels
def GetNodes(self, node_filter_dict=None, return_fields_list=None):
"""
Make a list of iotlab nodes and their properties from information
given by OAR. Search for specific nodes if some filters are
specified. Nodes properties returned if no return_fields_list given:
'hrn','archi','mobile','hostname','site','boot_state','node_id',
'radio','posx','posy','oar_id','posz'.
:param node_filter_dict: dictionnary of lists with node properties. For
instance, if you want to look for a specific node with its hrn,
the node_filter_dict should be {'hrn': [hrn_of_the_node]}
:type node_filter_dict: dict
:param return_fields_list: list of specific fields the user wants to be
returned.
:type return_fields_list: list
:returns: list of dictionaries with node properties
:rtype: list
"""
node_dict_by_id = self.oar.parser.SendRequest("GET_resources_full")
node_dict_list = node_dict_by_id.values()
logger.debug(" IOTLAB_API GetNodes node_filter_dict %s \
return_fields_list %s " % (node_filter_dict, return_fields_list))
#No filtering needed return the list directly
if not (node_filter_dict or return_fields_list):
return node_dict_list
return_node_list = []
if node_filter_dict:
for filter_key in node_filter_dict:
try:
#Filter the node_dict_list by each value contained in the
#list node_filter_dict[filter_key]
for value in node_filter_dict[filter_key]:
for node in node_dict_list:
if node[filter_key] == value:
if return_fields_list:
tmp = {}
for k in return_fields_list:
tmp[k] = node[k]
return_node_list.append(tmp)
else:
return_node_list.append(node)
except KeyError:
logger.log_exc("GetNodes KeyError")
return
return return_node_list
def exists(self, tablename):
"""
Checks if the table specified as tablename exists.
"""
try:
metadata = MetaData (bind=self.slab_engine)
table = Table (tablename, metadata, autoload=True)
return True
except NoSuchTableError:
logger.log_exc("SLABPOSTGRES tablename %s does not exists" \
%(tablename))
return False
def add_rule_to_group(self, group_name=None, protocol='tcp', cidr_ip='0.0.0.0/0',
port_range=None, icmp_type_code=None,
source_group_name=None, source_group_owner_id=None):
try:
from_port, to_port = self._validate_port_range(port_range)
icmp_type = self._validate_icmp_type_code(icmp_type_code)
if icmp_type and icmp_type[0] and icmp_type[1]:
from_port, to_port = icmp_type[0], icmp_type[1]
group = self.client.security_groups.find(name=group_name)
self.client.security_group_rules.create(group.id, \
protocol, from_port, to_port,cidr_ip)
except Exception, ex:
logger.log_exc("Failed to add rule to group %s" % group_name)
def LdapDelete(self, person_dn):
"""
Deletes a person in LDAP. Uses the dn of the user.
"""
#Connect and bind
result = self.conn.connect()
if(result['bool']):
try:
self.conn.ldapserv.delete_s(person_dn)
self.conn.close()
return {'bool': True}
except ldap.LDAPError, error:
logger.log_exc("LDAP Delete Error %s" %error)
return {'bool': False}
def __init__(self, manager, interface, config):
if isinstance(manager, ModuleType):
# old-fashioned module implementation
self.manager = manager
elif isinstance(manager, ClassType):
# create an instance; we don't pass the api in argument as it is passed
# to the actual method calls anyway
self.manager = manager(config)
else:
# that's what happens when there's something wrong with the db
# or any bad stuff of that kind at startup time
logger.log_exc(
"Failed to create a manager, startup sequence is broken")
raise SfaAPIError, "Argument to ManagerWrapper must be a module or class"
self.interface = interface
def __init__(self, config_file = "/etc/sfa/topology"):
set.__init__(self)
try:
# load the links
f = open(config_file, 'r')
for line in f:
ignore = line.find('#')
if ignore > -1:
line = line[0:ignore]
tup = line.split()
if len(tup) > 1:
self.add((tup[0], tup[1]))
except Exception, e:
logger.log_exc("Could not find or load the configuration file: %s" % config_file)
raise
def get_version(self):
### if we already know the answer:
if self.probed:
return self._version
### otherwise let's look in the cache file
logger.debug("searching in version cache %s" % self.url())
cached_version = VersionCache().get(self.url())
if cached_version is not None:
logger.info("Retrieved version info from cache %s" % self.url())
return cached_version
### otherwise let's do the hard work
# dummy to meet Sfi's expectations for its 'options' field
class DummyOptions:
pass
options = DummyOptions()
options.verbose = self.verbose
options.timeout = 10
try:
client = Sfi(options)
client.read_config()
client.bootstrap()
key_file = client.private_key
cert_file = client.my_gid
logger.debug("using key %s & cert %s" % (key_file, cert_file))
url = self.url()
logger.info('issuing GetVersion at %s' % url)
# setting timeout here seems to get the call to fail - even though the response time is fast
#server=SfaServerProxy(url, key_file, cert_file, verbose=self.verbose, timeout=options.timeout)
server = SfaServerProxy(url,
key_file,
cert_file,
verbose=self.verbose)
self._version = ReturnValue.get_value(server.GetVersion())
except:
logger.log_exc("failed to get version")
self._version = {}
# so that next run from this process will find out
self.probed = True
# store in version cache so next processes will remember for an hour
cache = VersionCache()
cache.set(self.url(), self._version)
cache.save()
logger.debug("Saved version for url=%s in version cache" % self.url())
# that's our result
return self._version
def LaunchExperimentOnOAR(self, added_nodes, slice_name, \
lease_start_time, lease_duration, slice_user=None):
"""
Create a job request structure based on the information provided
and post the job on OAR.
:param added_nodes: list of nodes that belong to the described lease.
:param slice_name: the slice hrn associated to the lease.
:param lease_start_time: timestamp of the lease startting time.
:param lease_duration: lease durationin minutes
"""
lease_dict = {}
lease_dict['lease_start_time'] = lease_start_time
lease_dict['lease_duration'] = lease_duration
lease_dict['added_nodes'] = added_nodes
lease_dict['slice_name'] = slice_name
lease_dict['slice_user'] = slice_user
lease_dict['grain'] = self.GetLeaseGranularity()
lease_dict['time_format'] = self.time_format
logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR slice_user %s\
\r\n " % (slice_user))
#Create the request for OAR
reqdict = self._create_job_structure_request_for_OAR(lease_dict)
# first step : start the OAR job and update the job
logger.debug("IOTLAB_API.PY \tLaunchExperimentOnOAR reqdict %s\
\r\n " % (reqdict))
answer = self.oar.POSTRequestToOARRestAPI('POST_job', \
reqdict, slice_user)
logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s " %
(answer))
try:
jobid = answer['id']
except KeyError:
logger.log_exc("IOTLAB_API \tLaunchExperimentOnOAR \
Impossible to create job %s " % (answer))
return None
if jobid:
logger.debug("IOTLAB_API \tLaunchExperimentOnOAR jobid %s \
added_nodes %s slice_user %s" %(jobid, added_nodes, \
slice_user))
return jobid
def POSTRequestToOARRestAPI(self, request, datadict, username=None):
""" Used to post a job on OAR , along with data associated
with the job.
"""
#first check that all params for are OK
try:
self.oarserver['uri'] = \
self.OAR_REQUEST_POST_URI_DICT[request]['uri']
except KeyError:
logger.log_exc("OARrestapi \tPOSTRequestToOARRestAPI request not \
valid")
return
if datadict and 'strval' in datadict:
self.oarserver['uri'] = self.oarserver['uri'].replace("id", \
str(datadict['strval']))
del datadict['strval']
data = json.dumps(datadict)
headers = {'X-REMOTE_IDENT':username, \
'content-type': self.POST_FORMAT['json']['content'], \
'content-length':str(len(data))}
try:
conn = HTTPConnection(self.oarserver['ip'], \
self.oarserver['port'])
conn.request("POST", self.oarserver['uri'], data, headers)
resp = conn.getresponse()
body = resp.read()
except NotConnected:
logger.log_exc("POSTRequestToOARRestAPI NotConnected ERROR: \
data %s \r\n \t\n \t\t headers %s uri %s" \
%(data,headers,self.oarserver['uri']))
except Exception as error:
logger.log_exc("POST_OAR_SERVER : Connection error: %s " % (error))
raise Exception("POST_OAR_SERVER : Connection error %s " % (error))
finally:
conn.close()
if resp.status >= 400:
raise ValueError("Response Error %s, %s" %
(resp.status, resp.reason))
try:
answer = json.loads(body)
logger.debug("POSTRequestToOARRestAPI : answer %s" % (answer))
return answer
except ValueError, error:
logger.log_exc("Failed to parse Server Response: error %s \
%s" % (error))
def exists(self, tablename):
"""
Checks if the table specified as tablename exists.
:param tablename: name of the table in the db that has to be checked.
:type tablename: string
:returns: True if the table exists, False otherwise.
:rtype: bool
"""
metadata = MetaData(bind=self.iotlab_engine)
try:
table = Table(tablename, metadata, autoload=True)
return True
except NoSuchTableError:
logger.log_exc("IOTLABPOSTGRES tablename %s does not exist" %
(tablename))
return False
def _Allocate(aggregate, server, xrn, credential, rspec, options):
tStart = time.time()
try:
# Need to call GetVersion at an aggregate to determine the supported
# rspec type/format beofre calling CreateSliver at an Aggregate.
#server_version = api.get_cached_server_version(server)
#if 'sfa' not in server_version and 'geni_api' in server_version:
# sfa aggregtes support both sfa and pg rspecs, no need to convert
# if aggregate supports sfa rspecs. otherwise convert to pg rspec
#rspec = RSpec(RSpecConverter.to_pg_rspec(rspec, 'request'))
#filter = {'component_manager_id': server_version['urn']}
#rspec.filter(filter)
#rspec = rspec.toxml()
result = server.Allocate(xrn, credential, rspec, options)
return {"aggregate": aggregate, "result": result, "elapsed": time.time()-tStart, "status": "success"}
except:
logger.log_exc('Something wrong in _Allocate with URL %s'%server.url)
return {"aggregate": aggregate, "elapsed": time.time()-tStart, "status": "exception", "exc_info": sys.exc_info()}
def _process_ldap_info_for_all_users(self, result_data):
"""Process the data of all enabled users in LDAP.
:param result_data: Contains information of all enabled users in LDAP
and is coming from LdapSearch.
:param result_data: list
.. seealso:: LdapSearch
"""
results = []
logger.debug(" LDAP.py _process_ldap_info_for_all_users result_data %s "
% (result_data))
for ldapentry in result_data:
logger.debug(" LDAP.py _process_ldap_info_for_all_users \
ldapentry name : %s " % (ldapentry[1]['uid'][0]))
tmpname = ldapentry[1]['uid'][0]
hrn = self.authname + "." + tmpname
tmpemail = ldapentry[1]['mail'][0]
if ldapentry[1]['mail'][0] == "unknown":
tmpemail = None
try:
results.append({
'type': 'user',
'pkey': ldapentry[1]['sshPublicKey'][0],
#'uid': ldapentry[1]['uid'][0],
'uid': tmpname ,
'email':tmpemail,
#'email': ldapentry[1]['mail'][0],
'first_name': ldapentry[1]['givenName'][0],
'last_name': ldapentry[1]['sn'][0],
#'phone': 'none',
'serial': 'none',
'authority': self.authname,
'peer_authority': '',
'pointer': -1,
'hrn': hrn,
})
except KeyError, error:
logger.log_exc("LDAPapi.PY \t LdapFindUser EXCEPTION %s"
% (error))
return
def add_rule_to_group(self,
group_name=None,
protocol='tcp',
cidr_ip='0.0.0.0/0',
port_range=None,
icmp_type_code=None,
source_group_name=None,
source_group_owner_id=None):
try:
from_port, to_port = self._validate_port_range(port_range)
icmp_type = self._validate_icmp_type_code(icmp_type_code)
if icmp_type and icmp_type[0] and icmp_type[1]:
from_port, to_port = icmp_type[0], icmp_type[1]
group = self.client.security_groups.find(name=group_name)
self.client.security_group_rules.create(group.id, \
protocol, from_port, to_port,cidr_ip)
except Exception, ex:
logger.log_exc("Failed to add rule to group %s" % group_name)
def filter_lease(reservation_list, filter_type, filter_value):
"""Filters the lease reservation list by removing each lease whose
filter_type is not equal to the filter_value provided. Returns the list
of leases in one slice, defined by the slice_hrn if filter_type
is 'slice_hrn'. Otherwise, returns all leases scheduled starting from
the filter_value if filter_type is 't_from'.
:param reservation_list: leases list
:type reservation_list: list of dictionary
:param filter_type: can be either 't_from' or 'slice hrn'
:type filter_type: string
:param filter_value: depending on the filter_type, can be the slice_hrn
or can be defining a timespan.
:type filter_value: if filter_type is 't_from', filter_value is int.
if filter_type is 'slice_hrn', filter_value is a string.
:returns: filtered_reservation_list, contains only leases running or
scheduled in the given slice (wanted_slice).Dict keys are
'lease_id','reserved_nodes','slice_id', 'state', 'user',
'component_id_list','slice_hrn', 'resource_ids', 't_from', 't_until'
:rtype: list of dict
"""
filtered_reservation_list = list(reservation_list)
logger.debug("IOTLAB_API \t filter_lease_name reservation_list %s" \
% (reservation_list))
try:
for reservation in reservation_list:
if \
(filter_type is 'slice_hrn' and \
reservation['slice_hrn'] != filter_value) or \
(filter_type is 't_from' and \
reservation['t_from'] > filter_value):
filtered_reservation_list.remove(reservation)
except TypeError:
logger.log_exc("Iotlabshell filter_lease : filter_type %s \
filter_value %s not in lease" %
(filter_type, filter_value))
return filtered_reservation_list
def LdapDelete(self, person_dn):
"""Deletes a person in LDAP. Uses the dn of the user.
:param person_dn: user's ldap dn.
:type person_dn: string
:returns: dictionary with bool True if successful, bool False
and the error if not.
:rtype: dict
"""
#Connect and bind
result = self.conn.connect()
if(result['bool']):
try:
self.conn.ldapserv.delete_s(person_dn)
self.conn.close()
return {'bool': True}
except ldap.LDAPError, error:
logger.log_exc("LDAP Delete Error %s" % error)
return {'bool': False, 'message': error}
def ParseJobsIds(self):
"""Associated with the GET_jobs_id OAR request.
Parses the json dict (OAR answer) to the GET_jobs_id request
/oarapi/jobs/id.json.
:returns: dictionary whose keys are listed in the local variable
job_resources and values that are in the json dictionary returned
by OAR with the job information.
:rtype: dict
"""
job_resources = [
'wanted_resources', 'name', 'id', 'start_time', 'state', 'owner',
'walltime', 'message'
]
# Unused variable providing the contents of the json dict returned from
# get job resources full request
job_resources_full = [
'launching_directory', 'links', 'resubmit_job_id', 'owner',
'events', 'message', 'scheduled_start', 'id', 'array_id',
'exit_code', 'properties', 'state', 'array_index', 'walltime',
'type', 'initial_request', 'stop_time', 'project', 'start_time',
'dependencies', 'api_timestamp', 'submission_time', 'reservation',
'stdout_file', 'types', 'cpuset_name', 'name', 'wanted_resources',
'queue', 'stderr_file', 'command'
]
job_info = self.json_page.raw_json
#logger.debug("OARESTAPI ParseJobsIds %s" %(self.json_page.raw_json))
values = []
try:
for k in job_resources:
values.append(job_info[k])
return dict(zip(job_resources, values))
except KeyError:
logger.log_exc("ParseJobsIds KeyError ")
def __call__(self, *args, **kwds):
"""
Main entry point for all SFA API functions. Type checks
arguments, authenticates, and executes call().
"""
try:
start = time.time()
methodname = self.name
if not self.api.interface or self.api.interface not in self.interfaces:
raise SfaInvalidAPIMethod(methodname, self.api.interface)
(min_args, max_args, defaults) = self.args()
# Check that the right number of arguments were passed in
if len(args) < len(min_args) or len(args) > len(max_args):
raise SfaInvalidArgumentCount(len(args), len(min_args), len(max_args))
for name, value, expected in zip(max_args, args, self.accepts):
self.type_check(name, value, expected, args)
logger.debug("method.__call__ [%s] : BEG %s"%(self.api.interface,methodname))
result = self.call(*args, **kwds)
runtime = time.time() - start
logger.debug("method.__call__ [%s] : END %s in %02f s (%s)"%\
(self.api.interface,methodname,runtime,getattr(self,'message',"[no-msg]")))
return result
except SfaFault, fault:
caller = ""
# Prepend caller and method name to expected faults
fault.faultString = caller + ": " + self.name + ": " + fault.faultString
runtime = time.time() - start
logger.log_exc("Method %s raised an exception"%self.name)
raise fault
def handler(req):
try:
if req.method != "POST":
req.content_type = "text/html"
req.send_http_header()
req.write("""
<html><head>
<title>SFA SliceMgr API XML-RPC/SOAP Interface</title>
</head><body>
<h1>SFA SliceMgr API XML-RPC/SOAP Interface</h1>
<p>Please use XML-RPC or SOAP to access the SFA API.</p>
</body></html>
""")
return apache.OK
# Read request
request = req.read(int(req.headers_in['content-length']))
# mod_python < 3.2: The IP address portion of remote_addr is
# incorrect (always 0.0.0.0) when IPv6 is enabled.
# http://issues.apache.org/jira/browse/MODPYTHON-64?page=all
(remote_ip, remote_port) = req.connection.remote_addr
remote_addr = (req.connection.remote_ip, remote_port)
# Handle request
response = api.handle(remote_addr, request)
# Write response
req.content_type = "text/xml; charset=" + api.encoding
req.send_http_header()
req.write(response)
return apache.OK
except Exception, err:
# Log error in /var/log/httpd/(ssl_)?error_log
logger.log_exc('%r'%err)
return apache.HTTP_INTERNAL_SERVER_ERROR
