def SendRequest(self, request, strval = None , username = None):
""" Connects to OAR , sends the valid GET requests and uses
the appropriate json parsing functions.
"""
self.raw_json = None
next_page = True
next_offset = None
save_json = None
self.concatenate = False
self.end = False
a = 0
save_json = []
self.raw_json_list = []
if request in self.OARrequests_uri_dict :
while next_page:
self.raw_json = self.server.GETRequestToOARRestAPI(request, \
strval, \
next_offset, \
username)
next_page , next_offset = self.FindNextPage()
if self.concatenate:
#self.raw_json_list.append(self.raw_json)
save_json.append(self.raw_json)
if self.concatenate and self.end :
#self.raw_json = self.ConcatenateJsonPages(self.raw_json_list)
self.raw_json = self.ConcatenateJsonPages(save_json)
return self.OARrequests_uri_dict[request]['parse_func'](self)
else:
logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " \
%(request))
def get_os_sliver_attributes(xml, filter=None):
if filter is None: filter={}
xpath = './openstack:*'
sliver_attrib_elems = xml.xpath(xpath)
sliver = OSSliver()
c = 0
for sliver_attrib_elem in sliver_attrib_elems:
tag = sliver_attrib_elem.tag.split('}')[-1]
if tag == 'availability_zone':
sliver['availability_zone'] = OSZone(sliver_attrib_elem.attrib, sliver_attrib_elem)
elif tag == 'security_group':
sliver['security_groups']=[]
sliver['security_groups'].append( OSSecGroup(sliver_attrib_elem.attrib, sliver_attrib_elem) )
sub_tags = sliver_attrib_elem.xpath('./openstack:rule')
rules=[]
if sub_tags and isinstance(sub_tags, list):
for sub_tag in sub_tags:
rules.append(OSSecGroupRule(sub_tag.attrib, sub_tag))
sliver['security_groups'][c]['rules'] = rules
c += 1
elif tag == 'flavor':
sliver['flavor'] = OSFlavor(sliver_attrib_elem.attrib, sliver_attrib_elem)
sub_tags = sliver_attrib_elem.xpath('./openstack:image')
if sub_tags and isinstance(sub_tags, list):
sliver['boot_image'] = OSImage(sub_tags[0].attrib, sub_tags[0])
elif tag == 'address':
pass
else:
logger.error("You should include essential information of Openstack sliver")
return sliver
def utcparse(input):
""" Translate a string into a time using dateutil.parser.parse but make sure it's in UTC time and strip
the timezone, so that it's compatible with normal datetime.datetime objects.
For safety this can also handle inputs that are either timestamps, or datetimes
"""
# prepare the input for the checks below by
# casting strings ('1327098335') to ints
if isinstance(input, StringTypes):
try:
input = int(input)
except ValueError:
pass
if isinstance (input, datetime.datetime):
logger.warn ("argument to utcparse already a datetime - doing nothing")
return input
elif isinstance (input, StringTypes):
t = dateutil.parser.parse(input)
if t.utcoffset() is not None:
t = t.utcoffset() + t.replace(tzinfo=None)
return t
elif isinstance (input, (int,float,long)):
return datetime.datetime.fromtimestamp(input)
else:
logger.error("Unexpected type in utcparse [%s]"%type(input))
def utcparse(input):
""" Translate a string into a time using dateutil.parser.parse but make sure it's in UTC time and strip
the timezone, so that it's compatible with normal datetime.datetime objects.
For safety this can also handle inputs that are either timestamps, or datetimes
"""
# prepare the input for the checks below by
# casting strings ('1327098335') to ints
if isinstance(input, StringTypes):
try:
input = int(input)
except ValueError:
pass
if isinstance(input, datetime.datetime):
logger.warn("argument to utcparse already a datetime - doing nothing")
return input
elif isinstance(input, StringTypes):
t = dateutil.parser.parse(input)
if t.utcoffset() is not None:
t = t.utcoffset() + t.replace(tzinfo=None)
return t
elif isinstance(input, (int, float, long)):
return datetime.datetime.fromtimestamp(input)
else:
logger.error("Unexpected type in utcparse [%s]" % type(input))
def init_server():
logger = logging.getLogger('EucaAggregate')
fileHandler = logging.FileHandler('/var/log/euca.log')
fileHandler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
logger.addHandler(fileHandler)
fileHandler.setLevel(logging.DEBUG)
logger.setLevel(logging.DEBUG)
configParser = ConfigParser()
configParser.read(['/etc/sfa/eucalyptus_aggregate.conf', 'eucalyptus_aggregate.conf'])
if len(configParser.sections()) < 1:
logger.error('No cloud defined in the config file')
raise Exception('Cannot find cloud definition in configuration file.')
# Only read the first section.
cloudSec = configParser.sections()[0]
cloud['name'] = cloudSec
cloud['access_key'] = configParser.get(cloudSec, 'access_key')
cloud['secret_key'] = configParser.get(cloudSec, 'secret_key')
cloud['cloud_url'] = configParser.get(cloudSec, 'cloud_url')
cloudURL = cloud['cloud_url']
if cloudURL.find('https://') >= 0:
cloudURL = cloudURL.replace('https://', '')
elif cloudURL.find('http://') >= 0:
cloudURL = cloudURL.replace('http://', '')
(cloud['ip'], parts) = cloudURL.split(':')
# Create image bundles
images = getEucaConnection().get_all_images()
cloud['images'] = images
cloud['imageBundles'] = {}
for i in images:
if i.type != 'machine' or i.kernel_id is None: continue
name = os.path.dirname(i.location)
detail = {'imageID' : i.id, 'kernelID' : i.kernel_id, 'ramdiskID' : i.ramdisk_id}
cloud['imageBundles'][name] = detail
# Initialize sqlite3 database and tables.
dbPath = '/etc/sfa/db'
dbName = 'euca_aggregate.db'
if not os.path.isdir(dbPath):
logger.info('%s not found. Creating directory ...' % dbPath)
os.mkdir(dbPath)
conn = connectionForURI('sqlite://%s/%s' % (dbPath, dbName))
sqlhub.processConnection = conn
Slice.createTable(ifNotExists=True)
EucaInstance.createTable(ifNotExists=True)
Meta.createTable(ifNotExists=True)
# Start the update process to keep track of the meta data
# about Eucalyptus instance.
Process(target=updateMeta).start()
# Make sure the schema exists.
if not os.path.exists(EUCALYPTUS_RSPEC_SCHEMA):
err = 'Cannot location schema at %s' % EUCALYPTUS_RSPEC_SCHEMA
logger.error(err)
raise Exception(err)
def getEucaConnection():
global cloud
accessKey = cloud['access_key']
secretKey = cloud['secret_key']
eucaURL = cloud['cloud_url']
useSSL = False
srvPath = '/'
eucaPort = 8773
logger = logging.getLogger('EucaAggregate')
if not accessKey or not secretKey or not eucaURL:
logger.error('Please set ALL of the required environment ' \
'variables by sourcing the eucarc file.')
return None
# Split the url into parts
if eucaURL.find('https://') >= 0:
useSSL = True
eucaURL = eucaURL.replace('https://', '')
elif eucaURL.find('http://') >= 0:
useSSL = False
eucaURL = eucaURL.replace('http://', '')
(eucaHost, parts) = eucaURL.split(':')
if len(parts) > 1:
parts = parts.split('/')
eucaPort = int(parts[0])
parts = parts[1:]
srvPath = '/'.join(parts)
return boto.connect_ec2(aws_access_key_id=accessKey,
aws_secret_access_key=secretKey,
is_secure=useSSL,
region=RegionInfo(None, 'eucalyptus', eucaHost),
port=eucaPort,
path=srvPath)
def SendRequest(self, request, strval=None, username=None):
""" Connects to OAR , sends the valid GET requests and uses
the appropriate json parsing functions.
:returns: calls to the appropriate parsing function, associated with the
GET request
:rtype: depends on the parsing function called.
.. seealso:: OARrequests_uri_dict
"""
save_json = None
self.json_page.ResetNextPage()
save_json = []
if request in self.OARrequests_uri_dict:
while self.json_page.next_page:
self.json_page.raw_json = self.server.GETRequestToOARRestAPI(
request, strval, self.json_page.next_offset, username)
self.json_page.FindNextPage()
if self.json_page.concatenate:
save_json.append(self.json_page.raw_json)
if self.json_page.concatenate and self.json_page.end:
self.json_page.raw_json = \
self.json_page.ConcatenateJsonPages(save_json)
return self.OARrequests_uri_dict[request]['parse_func'](self)
else:
logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST " %
(request))
def verify(self, trusted_certs=None, schema=None, trusted_certs_required=True):
if not self.xml:
self.decode()
# validate against RelaxNG schema
if HAVELXML:
if schema and os.path.exists(schema):
tree = etree.parse(StringIO(self.xml))
schema_doc = etree.parse(schema)
xmlschema = etree.XMLSchema(schema_doc)
if not xmlschema.validate(tree):
error = xmlschema.error_log.last_error
message = "%s: %s (line %s)" % (self.pretty_cred(), error.message, error.line)
raise CredentialNotVerifiable(message)
if trusted_certs_required and trusted_certs is None:
trusted_certs = []
# trusted_cert_objects = [GID(filename=f) for f in trusted_certs]
trusted_cert_objects = []
ok_trusted_certs = []
# If caller explicitly passed in None that means skip cert chain validation.
# Strange and not typical
if trusted_certs is not None:
for f in trusted_certs:
try:
# Failures here include unreadable files
# or non PEM files
trusted_cert_objects.append(GID(filename=f))
ok_trusted_certs.append(f)
except Exception, exc:
logger.error("Failed to load trusted cert from %s: %r"%( f, exc))
trusted_certs = ok_trusted_certs
def SendRequest(self, request, strval=None, username=None):
""" Connects to OAR , sends the valid GET requests and uses
the appropriate json parsing functions.
:returns: calls to the appropriate parsing function, associated with the
GET request
:rtype: depends on the parsing function called.
.. seealso:: OARrequests_uri_dict
"""
save_json = None
self.json_page.ResetNextPage()
save_json = []
if request in self.OARrequests_uri_dict:
while self.json_page.next_page:
self.json_page.raw_json = self.server.GETRequestToOARRestAPI(
request,
strval,
self.json_page.next_offset,
username)
self.json_page.FindNextPage()
if self.json_page.concatenate:
save_json.append(self.json_page.raw_json)
if self.json_page.concatenate and self.json_page.end:
self.json_page.raw_json = \
self.json_page.ConcatenateJsonPages(save_json)
return self.OARrequests_uri_dict[request]['parse_func'](self)
else:
logger.error("OARRESTAPI OARGetParse __init__ : ERROR_REQUEST "
% (request))
def reserveInstance(self, botoConn, pubKeys):
logger = logging.getLogger('EucaAggregate')
logger.info('Reserving an instance: image: %s, kernel: ' \
'%s, ramdisk: %s, type: %s, key: %s' % \
(self.image_id, self.kernel_id, self.ramdisk_id,
self.inst_type, self.key_pair))
# XXX The return statement is for testing. REMOVE in production
#return
try:
reservation = botoConn.run_instances(self.image_id,
kernel_id = self.kernel_id,
ramdisk_id = self.ramdisk_id,
instance_type = self.inst_type,
key_name = self.key_pair,
user_data = pubKeys)
for instance in reservation.instances:
self.instance_id = instance.id
# If there is an error, destroy itself.
except EC2ResponseError, ec2RespErr:
errTree = ET.fromstring(ec2RespErr.body)
msg = errTree.find('.//Message')
logger.error(msg.text)
self.destroySelf()
def merge_rspecs(rspecs):
"""
Merge merge a list of RSpecs into 1 RSpec, and return the result.
rspecs must be a valid RSpec string or list of RSpec strings.
"""
if not rspecs or not isinstance(rspecs, list):
return rspecs
# ugly hack to avoid sending the same info twice, when the call graph has dags
known_networks={}
def register_network (network):
try:
known_networks[network.get('name')]=True
except:
logger.error("merge_rspecs: cannot register network with no name in rspec")
pass
def is_registered_network (network):
try:
return network.get('name') in known_networks
except:
logger.error("merge_rspecs: cannot retrieve network with no name in rspec")
return False
# the resulting tree
rspec = None
for input_rspec in rspecs:
# ignore empty strings as returned with used call_ids
if not input_rspec: continue
try:
tree = etree.parse(StringIO(input_rspec))
except etree.XMLSyntaxError:
# consider failing silently here
logger.log_exc("merge_rspecs, parse error")
message = str(sys.exc_info()[1]) + ' with ' + input_rspec
raise InvalidRSpec(message)
root = tree.getroot()
if not root.get("type") in ["SFA"]:
logger.error("merge_rspecs: unexpected type for rspec root, %s"%root.get('type'))
continue
if rspec == None:
# we scan the first input, register all networks
# in addition we remove duplicates - needed until everyone runs 1.0-10
rspec = root
for network in root.iterfind("./network"):
if not is_registered_network(network):
register_network(network)
else:
# duplicate in the first input - trash it
root.remove(network)
else:
for network in root.iterfind("./network"):
if not is_registered_network(network):
rspec.append(deepcopy(network))
register_network(network)
for request in root.iterfind("./request"):
rspec.append(deepcopy(request))
return etree.tostring(rspec, xml_declaration=True, pretty_print=True)
def scan(self,interfaces,graph):
if not isinstance(interfaces,list):
interfaces=[interfaces]
# remember node to interface mapping
node2interface={}
# add entry points right away using the interface uid's as a key
to_scan=interfaces
for i in interfaces:
graph.add_node(i.uid())
node2interface[graph.get_node(i.uid())]=i
scanned=[]
# keep on looping until we reach a fixed point
# don't worry about abels and shapes that will get fixed later on
while to_scan:
for interface in to_scan:
# performing xmlrpc call
version=interface.get_version()
if self.verbose:
logger.info("GetVersion at interface %s"%interface.url())
if not version:
logger.info("<EMPTY GetVersion(); offline or cannot authenticate>")
else:
for (k,v) in version.iteritems():
if not isinstance(v,dict):
logger.info("\r\t%s:%s"%(k,v))
else:
logger.info(k)
for (k1,v1) in v.iteritems():
logger.info("\r\t\t%s:%s"%(k1,v1))
# 'geni_api' is expected if the call succeeded at all
# 'peers' is needed as well as AMs typically don't have peers
if 'geni_api' in version and 'peers' in version:
# proceed with neighbours
for (next_name,next_url) in version['peers'].iteritems():
next_interface=Interface(next_url)
# locate or create node in graph
try:
# if found, we're good with this one
next_node=graph.get_node(next_interface.uid())
except:
# otherwise, let's move on with it
graph.add_node(next_interface.uid())
next_node=graph.get_node(next_interface.uid())
node2interface[next_node]=next_interface
to_scan.append(next_interface)
graph.add_edge(interface.uid(),next_interface.uid())
scanned.append(interface)
to_scan.remove(interface)
# we've scanned the whole graph, let's get the labels and shapes right
for node in graph.nodes():
interface=node2interface.get(node,None)
if interface:
for (k,v) in interface.get_layout().iteritems():
node.attr[k]=v
else:
logger.error("MISSED interface with node %s"%node)
def scan(self, interfaces, graph):
if not isinstance(interfaces, list):
interfaces = [interfaces]
# remember node to interface mapping
node2interface = {}
# add entry points right away using the interface uid's as a key
to_scan = interfaces
for i in interfaces:
graph.add_node(i.uid())
node2interface[graph.get_node(i.uid())] = i
scanned = []
# keep on looping until we reach a fixed point
# don't worry about abels and shapes that will get fixed later on
while to_scan:
for interface in to_scan:
# performing xmlrpc call
logger.info("retrieving/fetching version at interface %s" %
interface.url())
version = interface.get_version()
if not version:
logger.info(
"<EMPTY GetVersion(); offline or cannot authenticate>")
else:
for (k, v) in version.iteritems():
if not isinstance(v, dict):
logger.debug("\r\t%s:%s" % (k, v))
else:
logger.debug(k)
for (k1, v1) in v.iteritems():
logger.debug("\r\t\t%s:%s" % (k1, v1))
# proceed with neighbours
if 'peers' in version:
for (next_name, next_url) in version['peers'].iteritems():
next_interface = Interface(
next_url, mentioned_in=interface.url())
# locate or create node in graph
try:
# if found, we're good with this one
next_node = graph.get_node(next_interface.uid())
except:
# otherwise, let's move on with it
graph.add_node(next_interface.uid())
next_node = graph.get_node(next_interface.uid())
node2interface[next_node] = next_interface
to_scan.append(next_interface)
graph.add_edge(interface.uid(), next_interface.uid())
scanned.append(interface)
to_scan.remove(interface)
# we've scanned the whole graph, let's get the labels and shapes right
for node in graph.nodes():
interface = node2interface.get(node, None)
if interface:
for (k, v) in interface.get_layout().iteritems():
node.attr[k] = v
else:
logger.error("MISSED interface with node %s" % node)
def set_expiration(self, expiration):
if isinstance(expiration, (int, float)):
self.expiration = datetime.datetime.fromtimestamp(expiration)
elif isinstance(expiration, datetime.datetime):
self.expiration = expiration
elif isinstance(expiration, StringTypes):
self.expiration = utcparse(expiration)
else:
logger.error("unexpected input type in Credential.set_expiration")
def set_expiration(self, expiration):
if isinstance(expiration, (int, float)):
self.expiration = datetime.datetime.fromtimestamp(expiration)
elif isinstance(expiration, datetime.datetime):
self.expiration = expiration
elif isinstance(expiration, StringTypes):
self.expiration = utcparse(expiration)
else:
logger.error("unexpected input type in Credential.set_expiration")
def LdapSearch(self, req_ldap=None, expected_fields=None):
"""
Used to search directly in LDAP, by using ldap filters and return
fields. When req_ldap is None, returns all the entries in the LDAP.
:param req_ldap: ldap style request, with appropriate filters,
example: (cn=*).
:param expected_fields: Fields in the user ldap entry that has to be
returned. If None is provided, will return 'mail', 'givenName',
'sn', 'uid', 'sshPublicKey', 'shadowExpire'.
:type req_ldap: string
:type expected_fields: list
.. seealso:: make_ldap_filters_from_record for req_ldap format.
"""
result = self.conn.connect(bind=False)
if (result['bool']):
return_fields_list = []
if expected_fields is None:
return_fields_list = ['mail', 'givenName', 'sn', 'uid',
'sshPublicKey', 'shadowExpire']
else:
return_fields_list = expected_fields
#No specifc request specified, get the whole LDAP
if req_ldap is None:
req_ldap = '(cn=*)'
logger.debug("LDAP.PY \t LdapSearch req_ldap %s \
return_fields_list %s" \
%(req_ldap, return_fields_list))
try:
msg_id = self.conn.ldapserv.search(
self.baseDN, ldap.SCOPE_SUBTREE,
req_ldap, return_fields_list)
#Get all the results matching the search from ldap in one
#shot (1 value)
result_type, result_data = \
self.conn.ldapserv.result(msg_id, 1)
self.conn.close()
logger.debug("LDAP.PY \t LdapSearch result_data %s"
% (result_data))
return result_data
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapSearch Error %s" % error)
return []
else:
logger.error("LDAP.PY \t Connection Failed")
return
def add_sliver_attribute(self, component_id, name, value, network=None):
nodes = self.get_nodes({'component_id': '*%s*' % component_id})
if nodes is not None and isinstance(nodes, list) and len(nodes) > 0:
node = nodes[0]
slivers = SFAv1Sliver.get_slivers(node)
if slivers:
sliver = slivers[0]
SFAv1Sliver.add_sliver_attribute(sliver, name, value)
else:
# should this be an assert / raise an exception?
logger.error("WARNING: failed to find component_id %s" % component_id)
def checkCredentialsSpeaksFor (self, *args, **kwds):
if 'options' not in kwds:
logger.error ("checkCredentialsSpeaksFor was not passed options=options")
return
# remove the options arg
options=kwds['options']; del kwds['options']
# compute the speaking_for_xrn arg and pass it to checkCredentials
if options is None: speaking_for_xrn=None
else: speaking_for_xrn=options.get('geni_speaking_for',None)
kwds['speaking_for_xrn']=speaking_for_xrn
return self.checkCredentials (*args, **kwds)
def _validate_icmp_type_code(self, icmp_type_code):
from_port = to_port = None
if isinstance(icmp_type_code, str):
code_parts = icmp_type_code.split(':')
if len(code_parts) > 1:
try:
from_port = int(code_parts[0])
to_port = int(code_parts[1])
except ValueError:
logger.error('port must be an integer.')
return (from_port, to_port)
def _validate_icmp_type_code(self, icmp_type_code):
from_port = to_port = None
if isinstance(icmp_type_code, str):
code_parts = icmp_type_code.split(':')
if len(code_parts) > 1:
try:
from_port = int(code_parts[0])
to_port = int(code_parts[1])
except ValueError:
logger.error('port must be an integer.')
return (from_port, to_port)
def add_sliver_attribute(self, component_id, name, value, network=None):
nodes = self.get_nodes({'component_id': '*%s*' % component_id})
if nodes is not None and isinstance(nodes, list) and len(nodes) > 0:
node = nodes[0]
slivers = SFAv1Sliver.get_slivers(node)
if slivers:
sliver = slivers[0]
SFAv1Sliver.add_sliver_attribute(sliver, name, value)
else:
# should this be an assert / raise an exception?
logger.error("WARNING: failed to find component_id %s" %
component_id)
def list_slices (self, creds, options):
vct_urns = []
try:
vcts = self.repo.list_entities(Vct)
#logger.debug("***********************found VCTs: %s " % vcts)
for vct in vcts:
logger.debug("***********************VCT commonName: %s | urn: %s" %(vct.commonName, hrn_to_urn(vct.commonName,'slice')))
vct_urns.append(hrn_to_urn(vct.commonName,'slice'))
except NoEntityFound:
logger.error("***********************No VCTs found")
logger.debug("***********************VCT_urns: %s" % vct_urns)
return vct_urns
def stop_slice (self, slice_urn, slice_hrn, creds):
logger.debug("***********************stop_slice() called")
logger.debug("***********************slice_urn: %s " % slice_urn)
vct_name = slice_hrn.rpartition('.')[2]
try:
vct = self.repo.get_unique_entity(Vct, commonName = vct_name)
self.__rp.stop(vct)
logger.debug("***********************stopped VCT: %s " % vct_name)
return 1
except NoEntityFound:
logger.error("***********************No VCT found to stop: %s " % vct_name)
return 0
def LdapGenerateUniqueLogin(self, record):
"""
Generate login for adding a new user in LDAP Directory
(four characters minimum length). Get proper last name and
first name so that the user's login can be generated.
:param record: Record must contain first_name and last_name.
:type record: dict
:returns: the generated login for the user described with record if the
login generation is successful, None if it fails.
:rtype: string or None
"""
#For compatibility with other ldap func
if 'mail' in record and 'email' not in record:
record['email'] = record['mail']
lower_first_name, lower_last_name = \
self.login_pwd.get_user_firstname_lastname(record)
index, login = self.login_pwd.choose_sets_chars_for_login(
lower_first_name, lower_last_name)
login_filter = '(uid=' + login + ')'
get_attrs = ['uid']
try:
#Check if login already in use
while (len(self.LdapSearch(login_filter, get_attrs)) is not 0):
index += 1
if index >= 9:
logger.error("LoginException : Generation login error \
with minimum four characters")
else:
try:
login = \
lower_first_name[0:index] + \
lower_last_name[0:
self.login_pwd.login_max_length
- index]
login_filter = '(uid=' + login + ')'
except KeyError:
print "lower_first_name - lower_last_name too short"
logger.debug("LDAP.API \t LdapGenerateUniqueLogin login %s"
% (login))
return login
except ldap.LDAPError, error:
logger.log_exc("LDAP LdapGenerateUniqueLogin Error %s" % (error))
return None
def LdapModifyUser(self, user_record, new_attributes_dict):
"""
Gets the record from one user based on the user sfa recordand changes
the attributes according to the specified new_attributes. Do not use
this if we need to modify the uid. Use a ModRDN operation instead
( modify relative DN ).
:param user_record: sfa user record.
:param new_attributes_dict: new user attributes, keys must be the
same as the LDAP model.
:type user_record: dict
:type new_attributes_dict: dict
:returns: bool True if successful, bool False if not.
:rtype: dict
.. seealso:: make_ldap_filters_from_record for info on what is mandatory
in the user_record.
.. seealso:: make_ldap_attributes_from_record for the LDAP objectclass.
"""
if user_record is None:
logger.error("LDAP \t LdapModifyUser Need user record ")
return {'bool': False}
#Get all the attributes of the user_uid_login
#person = self.LdapFindUser(record_filter,[])
req_ldap = self.make_ldap_filters_from_record(user_record)
person_list = self.LdapSearch(req_ldap, [])
logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s"
% (person_list))
if person_list and len(person_list) > 1:
logger.error("LDAP \t LdapModifyUser Too many users returned")
return {'bool': False}
if person_list is None:
logger.error("LDAP \t LdapModifyUser User %s doesn't exist "
% (user_record))
return {'bool': False}
# The dn of our existing entry/object
#One result only from ldapSearch
person = person_list[0][1]
dn = 'uid=' + person['uid'][0] + "," + self.baseDN
if new_attributes_dict:
old = {}
for k in new_attributes_dict:
if k not in person:
old[k] = ''
else:
old[k] = person[k]
logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"
% (new_attributes_dict))
result = self.LdapModify(dn, old, new_attributes_dict)
return result
else:
logger.error("LDAP \t LdapModifyUser No new attributes given. ")
return {'bool': False}
def __init__(self, create=False, subject=None, string=None, filename=None, cred=None):
self.gidCaller = None
self.gidObject = None
self.expiration = None
self.privileges = None
self.issuer_privkey = None
self.issuer_gid = None
self.issuer_pubkey = None
self.parent = None
self.signature = None
self.xml = None
self.refid = None
self.type = None
self.version = None
if cred:
if isinstance(cred, StringTypes):
string = cred
self.type = 'geni_sfa'
self.version = '1.0'
elif isinstance(cred, dict):
string = cred['geni_value']
self.type = cred['geni_type']
self.version = cred['geni_version']
if string or filename:
if string:
str = string
elif filename:
str = file(filename).read()
# if this is a legacy credential, write error and bail out
if isinstance (str, StringTypes) and str.strip().startswith("-----"):
logger.error("Legacy credentials not supported any more - giving up with %s..."%str[:10])
return
else:
self.xml = str
self.decode()
# Find an xmlsec1 path
self.xmlsec_path = ''
paths = ['/usr/bin','/usr/local/bin','/bin','/opt/bin','/opt/local/bin']
for path in paths:
if os.path.isfile(path + '/' + 'xmlsec1'):
self.xmlsec_path = path + '/' + 'xmlsec1'
break
def LdapSearch (self, req_ldap = None, expected_fields = None ):
"""
Used to search directly in LDAP, by using ldap filters and
return fields.
When req_ldap is None, returns all the entries in the LDAP.
"""
result = self.conn.connect(bind = False)
if (result['bool']) :
return_fields_list = []
if expected_fields == None :
return_fields_list = ['mail','givenName', 'sn', 'uid', \
'sshPublicKey', 'shadowExpire']
else :
return_fields_list = expected_fields
#No specifc request specified, get the whole LDAP
if req_ldap == None:
req_ldap = '(cn=*)'
logger.debug("LDAP.PY \t LdapSearch req_ldap %s \
return_fields_list %s" \
%(req_ldap, return_fields_list))
try:
msg_id = self.conn.ldapserv.search(
self.baseDN,ldap.SCOPE_SUBTREE,\
req_ldap, return_fields_list)
#Get all the results matching the search from ldap in one
#shot (1 value)
result_type, result_data = \
self.conn.ldapserv.result(msg_id,1)
self.conn.close()
logger.debug("LDAP.PY \t LdapSearch result_data %s"\
%(result_data))
return result_data
except ldap.LDAPError,error :
logger.log_exc("LDAP LdapSearch Error %s" %error)
return []
else:
logger.error("LDAP.PY \t Connection Failed" )
return
def utcparse(input):
""" Translate a string into a time using dateutil.parser.parse but make sure it's in UTC time and strip
the timezone, so that it's compatible with normal datetime.datetime objects.
For safety this can also handle inputs that are either timestamps, or datetimes
"""
def handle_shorthands (input):
"""recognize string like +5d or +3w or +2m as
2 days, 3 weeks or 2 months from now"""
if input.startswith('+'):
match=re.match (r"([0-9]+)([dwm])",input[1:])
if match:
how_many=int(match.group(1))
what=match.group(2)
if what == 'd': d=datetime.timedelta(days=how_many)
elif what == 'w': d=datetime.timedelta(weeks=how_many)
elif what == 'm': d=datetime.timedelta(weeks=4*how_many)
return datetime.datetime.utcnow()+d
# prepare the input for the checks below by
# casting strings ('1327098335') to ints
if isinstance(input, StringTypes):
try:
input = int(input)
except ValueError:
try:
new_input=handle_shorthands(input)
if new_input is not None: input=new_input
except:
import traceback
traceback.print_exc()
#################### here we go
if isinstance (input, datetime.datetime):
#logger.info ("argument to utcparse already a datetime - doing nothing")
return input
elif isinstance (input, StringTypes):
t = dateutil.parser.parse(input)
if t.utcoffset() is not None:
t = t.utcoffset() + t.replace(tzinfo=None)
return t
elif isinstance (input, (int,float,long)):
return datetime.datetime.fromtimestamp(input)
else:
logger.error("Unexpected type in utcparse [%s]"%type(input))
def verify(self, trusted_certs):
if not self.xml:
self.decode()
# trusted_cert_objects = [GID(filename=f) for f in trusted_certs]
trusted_cert_objects = []
ok_trusted_certs = []
for f in trusted_certs:
try:
# Failures here include unreadable files
# or non PEM files
trusted_cert_objects.append(GID(filename=f))
ok_trusted_certs.append(f)
except Exception, exc:
import traceback
logger.error("Failed to load trusted cert from %s: %r", f, exc)
logger.debug(traceback.format_exc(exc))
def toXML(self):
logger = logging.getLogger('EucaAggregate')
if not self.cloudInfo:
logger.error('No cloud information')
return ''
xml = self.eucaRSpec
cloud = self.cloudInfo
with xml.RSpec(type='eucalyptus'):
with xml.network(name=cloud['name']):
with xml.ipv4:
xml << cloud['ip']
#self.__keyPairsXML(cloud['keypairs'])
#self.__imagesXML(cloud['images'])
self.__imageBundleXML(cloud['imageBundles'])
self.__clustersXML(cloud['clusters'])
return str(xml)
def verify(self, trusted_certs):
if not self.xml:
self.decode()
# trusted_cert_objects = [GID(filename=f) for f in trusted_certs]
trusted_cert_objects = []
ok_trusted_certs = []
for f in trusted_certs:
try:
# Failures here include unreadable files
# or non PEM files
trusted_cert_objects.append(GID(filename=f))
ok_trusted_certs.append(f)
except Exception, exc:
import traceback
logger.error("Failed to load trusted cert from %s: %r", f, exc)
logger.debug(traceback.format_exc(exc))
def choose_sets_chars_for_login(self, lower_first_name, lower_last_name):
"""
Algorithm to select sets of characters from the first name and last
name, depending on the lenght of the last name and the maximum login
length which in our case is set to 8 characters.
:param lower_first_name: user's first name in lower case.
:param lower_last_name: usr's last name in lower case.
:returns: user's login
:rtype: string
"""
length_last_name = len(lower_last_name)
self.login_max_length = 8
#Try generating a unique login based on first name and last name
if length_last_name >= self.login_max_length:
login = lower_last_name[0:self.login_max_length]
index = 0
logger.debug("login : %s index : %s" % (login, index))
elif length_last_name >= 4:
login = lower_last_name
index = 0
logger.debug("login : %s index : %s" % (login, index))
elif length_last_name == 3:
login = lower_first_name[0:1] + lower_last_name
index = 1
logger.debug("login : %s index : %s" % (login, index))
elif length_last_name == 2:
if len(lower_first_name) >= 2:
login = lower_first_name[0:2] + lower_last_name
index = 2
logger.debug("login : %s index : %s" % (login, index))
else:
logger.error("LoginException : \
Generation login error with \
minimum four characters")
else:
logger.error("LDAP LdapGenerateUniqueLogin failed : \
impossible to generate unique login for %s %s"
% (lower_first_name, lower_last_name))
return index, login
def __init__ (self, api):
config = api.config
flavour = config.SFA_GENERIC_FLAVOUR
# to be cleaned
if flavour == "nitos":
from sfa.nitos.nitosdriver import NitosDriver
self.driver = NitosDriver(api)
elif flavour == "fd":
from sfa.federica.fddriver import FdDriver
self.driver = FdDriver(api)
else:
logger.error("V2ToV3Adapter: Unknown Flavour !!!\n Supported Flavours: nitos, fd")
# Caching
if config.SFA_AGGREGATE_CACHING:
if self.driver.cache:
self.cache = self.driver.cache
else:
self.cache = Cache()
def main():
usage="%prog: trash the registry DB (the 'sfa' table in the 'planetlab5' database)"
parser = OptionParser(usage=usage)
parser.add_option('-f','--file-system',dest='clean_fs',action='store_true',default=False,
help='Clean up the /var/lib/sfa/authorities area as well')
parser.add_option('-c','--certs',dest='clean_certs',action='store_true',default=False,
help='Remove all cached certs/gids found in /var/lib/sfa/authorities area as well')
(options,args)=parser.parse_args()
if args:
parser.print_help()
sys.exit(1)
logger.info("Purging SFA records from database")
table = SfaTable()
table.sfa_records_purge()
if options.clean_certs:
# remove the server certificate and all gids found in /var/lib/sfa/authorities
logger.info("Purging cached certificates")
for (dir, _, files) in os.walk('/var/lib/sfa/authorities'):
for file in files:
if file.endswith('.gid') or file == 'server.cert':
path=dir+os.sep+file
os.unlink(path)
if not os.path.exists(path):
logger.info("Unlinked file %s"%path)
else:
logger.error("Could not unlink file %s"%path)
if options.clean_fs:
# just remove all files that do not match 'server.key' or 'server.cert'
logger.info("Purging registry filesystem cache")
preserved_files = [ 'server.key', 'server.cert']
for (dir,_,files) in os.walk('/var/lib/sfa/authorities'):
for file in files:
if file in preserved_files: continue
path=dir+os.sep+file
os.unlink(path)
if not os.path.exists(path):
logger.info("Unlinked file %s"%path)
else:
logger.error("Could not unlink file %s"%path)
def __init__(self, api):
config = api.config
flavour = config.SFA_GENERIC_FLAVOUR
# to be cleaned
if flavour == "nitos":
from sfa.nitos.nitosdriver import NitosDriver
self.driver = NitosDriver(api)
elif flavour == "fd":
from sfa.federica.fddriver import FdDriver
self.driver = FdDriver(api)
else:
logger.error(
"V2ToV3Adapter: Unknown Flavour !!!\n Supported Flavours: nitos, fd"
)
# Caching
if config.SFA_AGGREGATE_CACHING:
if self.driver.cache:
self.cache = self.driver.cache
else:
self.cache = Cache()
def GetSites(self, site_filter_name_list=None, return_fields_list=None):
"""Returns the list of Iotlab's sites with the associated nodes and
the sites' properties as dictionaries.
Site properties:
['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']
Uses the OAR request GET_sites to find the Iotlab's sites.
:param site_filter_name_list: used to specify specific sites
:param return_fields_list: field that has to be returned
:type site_filter_name_list: list
:type return_fields_list: list
"""
site_dict = self.oar.parser.SendRequest("GET_sites")
#site_dict : dict where the key is the sit ename
return_site_list = []
if not (site_filter_name_list or return_fields_list):
return_site_list = site_dict.values()
return return_site_list
for site_filter_name in site_filter_name_list:
if site_filter_name in site_dict:
if return_fields_list:
for field in return_fields_list:
tmp = {}
try:
tmp[field] = site_dict[site_filter_name][field]
except KeyError:
logger.error("GetSites KeyError %s " % (field))
return None
return_site_list.append(tmp)
else:
return_site_list.append(site_dict[site_filter_name])
return return_site_list
def GetSites(self, site_filter_name_list=None, return_fields_list=None):
"""Returns the list of Iotlab's sites with the associated nodes and
the sites' properties as dictionaries.
Site properties:
['address_ids', 'slice_ids', 'name', 'node_ids', 'url', 'person_ids',
'site_tag_ids', 'enabled', 'site', 'longitude', 'pcu_ids',
'max_slivers', 'max_slices', 'ext_consortium_id', 'date_created',
'latitude', 'is_public', 'peer_site_id', 'peer_id', 'abbreviated_name']
Uses the OAR request GET_sites to find the Iotlab's sites.
:param site_filter_name_list: used to specify specific sites
:param return_fields_list: field that has to be returned
:type site_filter_name_list: list
:type return_fields_list: list
"""
site_dict = self.oar.parser.SendRequest("GET_sites")
#site_dict : dict where the key is the sit ename
return_site_list = []
if not (site_filter_name_list or return_fields_list):
return_site_list = site_dict.values()
return return_site_list
for site_filter_name in site_filter_name_list:
if site_filter_name in site_dict:
if return_fields_list:
for field in return_fields_list:
tmp = {}
try:
tmp[field] = site_dict[site_filter_name][field]
except KeyError:
logger.error("GetSites KeyError %s " % (field))
return None
return_site_list.append(tmp)
else:
return_site_list.append(site_dict[site_filter_name])
return return_site_list
def verify(self,
trusted_certs=None,
schema=None,
trusted_certs_required=True):
if not self.xml:
self.decode()
# validate against RelaxNG schema
if HAVELXML and not self.legacy:
if schema and os.path.exists(schema):
tree = etree.parse(StringIO(self.xml))
schema_doc = etree.parse(schema)
xmlschema = etree.XMLSchema(schema_doc)
if not xmlschema.validate(tree):
error = xmlschema.error_log.last_error
message = "%s: %s (line %s)" % (
self.get_summary_tostring(), error.message, error.line)
raise CredentialNotVerifiable(message)
if trusted_certs_required and trusted_certs is None:
trusted_certs = []
# trusted_cert_objects = [GID(filename=f) for f in trusted_certs]
trusted_cert_objects = []
ok_trusted_certs = []
# If caller explicitly passed in None that means skip cert chain validation.
# Strange and not typical
if trusted_certs is not None:
for f in trusted_certs:
try:
# Failures here include unreadable files
# or non PEM files
trusted_cert_objects.append(GID(filename=f))
ok_trusted_certs.append(f)
except Exception, exc:
logger.error("Failed to load trusted cert from %s: %r" %
(f, exc))
trusted_certs = ok_trusted_certs
def execute(self, query, params = None):
cursor = self.cursor()
try:
# psycopg2 requires %()s format for all parameters,
# regardless of type.
# this needs to be done carefully though as with pattern-based filters
# we might have percents embedded in the query
# so e.g. GetPersons({'email':'*fake*'}) was resulting in .. LIKE '%sake%'
if psycopg2:
query = re.sub(r'(%\([^)]*\)|%)[df]', r'\1s', query)
# rewrite wildcards set by Filter.py as '***' into '%'
query = query.replace ('***','%')
if not params:
if self.debug:
logger.debug('execute0 %r'%query)
cursor.execute(query)
elif isinstance(params,dict):
if self.debug:
logger.debug('execute-dict: params=[%r] query=[%r]'%(params,query%params))
cursor.execute(query,params)
elif isinstance(params,tuple) and len(params)==1:
if self.debug:
logger.debug('execute-tuple %r'%(query%params[0]))
cursor.execute(query,params[0])
else:
param_seq=(params,)
if self.debug:
for params in param_seq:
logger.debug('executemany %r'%(query%params))
cursor.executemany(query, param_seq)
(self.rowcount, self.description, self.lastrowid) = \
(cursor.rowcount, cursor.description, cursor.lastrowid)
except Exception, e:
try:
self.rollback()
except:
pass
uuid = commands.getoutput("uuidgen")
logger.error("Database error %s:" % uuid)
logger.error("Exception=%r"%e)
logger.error("Query=%r"%query)
logger.error("Params=%r"%pformat(params))
logger.log_exc("PostgreSQL.execute caught exception")
raise SfaDBError("Please contact support: %s" % str(e))
def LdapModifyUser(self, user_record, new_attributes_dict):
"""
Gets the record from one user_uid_login based on record_filter
and changes the attributes according to the specified new_attributes.
Does not use this if we need to modify the uid. Use a ModRDN
#operation instead ( modify relative DN )
"""
if user_record is None:
logger.error("LDAP \t LdapModifyUser Need user record ")
return {'bool': False}
#Get all the attributes of the user_uid_login
#person = self.LdapFindUser(record_filter,[])
req_ldap = self.make_ldap_filters_from_record(user_record)
person_list = self.LdapSearch(req_ldap,[])
logger.debug("LDAPapi.py \t LdapModifyUser person_list : %s" \
%(person_list))
if person_list and len(person_list) > 1 :
logger.error("LDAP \t LdapModifyUser Too many users returned")
return {'bool': False}
if person_list is None :
logger.error("LDAP \t LdapModifyUser User %s doesn't exist "\
%(user_record))
return {'bool': False}
# The dn of our existing entry/object
#One result only from ldapSearch
person = person_list[0][1]
dn = 'uid=' + person['uid'][0] + "," +self.baseDN
if new_attributes_dict:
old = {}
for k in new_attributes_dict:
if k not in person:
old[k] = ''
else :
old[k] = person[k]
logger.debug(" LDAPapi.py \t LdapModifyUser new_attributes %s"\
%( new_attributes_dict))
result = self.LdapModify(dn, old,new_attributes_dict)
return result
else:
logger.error("LDAP \t LdapModifyUser No new attributes given. ")
return {'bool': False}
