def update_cert_records(gids):
"""
Make sure there is a record in the registry for the specified gids.
Removes old records from the db.
"""
# import db stuff here here so this module can be loaded by PlcComponentApi
from sfa.storage.alchemy import dbsession
from sfa.storage.model import RegRecord
if not gids:
return
# get records that actually exist in the db
gid_urns = [gid.get_urn() for gid in gids]
hrns_expected = [gid.get_hrn() for gid in gids]
records_found = dbsession.query(RegRecord).\
filter_by(pointer=-1).filter(RegRecord.hrn.in_(hrns_expected)).all()
# remove old records
for record in records_found:
if record.hrn not in hrns_expected and \
record.hrn != self.api.config.SFA_INTERFACE_HRN:
dbsession.delete(record)
# TODO: store urn in the db so we do this in 1 query
for gid in gids:
hrn, type = gid.get_hrn(), gid.get_type()
record = dbsession.query(RegRecord).filter_by(hrn=hrn, type=type,pointer=-1).first()
if not record:
record = RegRecord (dict= {'type':type,
'hrn': hrn,
'authority': get_authority(hrn),
'gid': gid.save_to_string(save_parents=True),
})
dbsession.add(record)
dbsession.commit()
def GetCredential(self, api, xrn, type, caller_xrn=None):
# convert xrn to hrn
if type:
hrn = urn_to_hrn(xrn)[0]
else:
hrn, type = urn_to_hrn(xrn)
# Is this a root or sub authority
auth_hrn = api.auth.get_authority(hrn)
if not auth_hrn or hrn == api.config.SFA_INTERFACE_HRN:
auth_hrn = hrn
auth_info = api.auth.get_auth_info(auth_hrn)
# get record info
record=dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first()
if not record:
raise RecordNotFound("hrn=%s, type=%s"%(hrn,type))
# get the callers gid
# if caller_xrn is not specified assume the caller is the record
# object itself.
if not caller_xrn:
caller_hrn = hrn
caller_gid = record.get_gid_object()
else:
caller_hrn, caller_type = urn_to_hrn(caller_xrn)
if caller_type:
caller_record = dbsession.query(RegRecord).filter_by(hrn=caller_hrn,type=caller_type).first()
else:
caller_record = dbsession.query(RegRecord).filter_by(hrn=caller_hrn).first()
if not caller_record:
raise RecordNotFound("Unable to associated caller (hrn=%s, type=%s) with credential for (hrn: %s, type: %s)"%(caller_hrn, caller_type, hrn, type))
caller_gid = GID(string=caller_record.gid)
object_hrn = record.get_gid_object().get_hrn()
# call the builtin authorization/credential generation engine
rights = api.auth.determine_user_rights(caller_hrn, record)
# make sure caller has rights to this object
if rights.is_empty():
raise PermissionError("%s has no rights to %s (%s)" % \
(caller_hrn, object_hrn, xrn))
object_gid = GID(string=record.gid)
new_cred = Credential(subject = object_gid.get_subject())
new_cred.set_gid_caller(caller_gid)
new_cred.set_gid_object(object_gid)
new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
#new_cred.set_pubkey(object_gid.get_pubkey())
new_cred.set_privileges(rights)
new_cred.get_privileges().delegate_all_privileges(True)
if hasattr(record,'expires'):
date = utcparse(record.expires)
expires = datetime_to_epoch(date)
new_cred.set_expiration(int(expires))
auth_kind = "authority,ma,sa"
# Parent not necessary, verify with certs
#new_cred.set_parent(api.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))
new_cred.encode()
new_cred.sign()
return new_cred.save_to_string(save_parents=True)
def update_pis (self, pi_hrns):
# don't ruin the import of that file in a client world
from sfa.storage.alchemy import dbsession
# strip that in case we have <researcher> words </researcher>
pi_hrns = [ x.strip() for x in pi_hrns ]
request = dbsession.query (RegUser).filter(RegUser.hrn.in_(pi_hrns))
logger.info ("RegAuthority.update_pis: %d incoming pis, %d matches found"%(len(pi_hrns),request.count()))
pis = dbsession.query (RegUser).filter(RegUser.hrn.in_(pi_hrns)).all()
self.reg_pis = pis
def update_researchers (self, researcher_hrns):
# don't ruin the import of that file in a client world
from sfa.storage.alchemy import dbsession
# strip that in case we have <researcher> words </researcher>
researcher_hrns = [ x.strip() for x in researcher_hrns ]
request = dbsession.query (RegUser).filter(RegUser.hrn.in_(researcher_hrns))
logger.info ("RegSlice.update_researchers: %d incoming researchers, %d matches found"%(len(researcher_hrns),request.count()))
researchers = dbsession.query (RegUser).filter(RegUser.hrn.in_(researcher_hrns)).all()
self.reg_researchers = researchers
def List (self, api, xrn, origin_hrn=None, options={}):
# load all know registry names into a prefix tree and attempt to find
# the longest matching prefix
hrn, type = urn_to_hrn(xrn)
registries = api.registries
registry_hrns = registries.keys()
tree = prefixTree()
tree.load(registry_hrns)
registry_hrn = tree.best_match(hrn)
#if there was no match then this record belongs to an unknow registry
if not registry_hrn:
raise MissingAuthority(xrn)
# if the best match (longest matching hrn) is not the local registry,
# forward the request
record_dicts = []
if registry_hrn != api.hrn:
credential = api.getCredential()
interface = api.registries[registry_hrn]
server_proxy = api.server_proxy(interface, credential)
record_list = server_proxy.List(xrn, credential, options)
# same as above, no need to process what comes from through xmlrpc
# pass foreign records as-is
record_dicts = record_list
# if we still have not found the record yet, try the local registry
# logger.debug("before trying local records, %d foreign records"% len(record_dicts))
if not record_dicts:
recursive = False
if ('recursive' in options and options['recursive']):
recursive = True
elif hrn.endswith('*'):
hrn = hrn[:-1]
recursive = True
if not api.auth.hierarchy.auth_exists(hrn):
raise MissingAuthority(hrn)
if recursive:
records = dbsession.query(RegRecord).filter(RegRecord.hrn.startswith(hrn)).all()
# logger.debug("recursive mode, found %d local records"%(len(records)))
else:
records = dbsession.query(RegRecord).filter_by(authority=hrn).all()
# logger.debug("non recursive mode, found %d local records"%(len(records)))
# so that sfi list can show more than plain names...
for record in records: augment_with_sfa_builtins (record)
record_dicts=[ record.todict(exclude_types=[InstrumentedList]) for record in records ]
return record_dicts
def get_pis (self):
# don't ruin the import of that file in a client world
from sfa.storage.alchemy import dbsession
from sfa.util.xrn import get_authority
authority_hrn = get_authority(self.hrn)
auth_record = dbsession.query(RegAuthority).filter_by(hrn=authority_hrn).first()
return auth_record.reg_pis
def fill_record_sfa_info(self, records):
def startswith(prefix, values):
return [value for value in values if value.startswith(prefix)]
# get user ids
user_ids = []
for record in records:
user_ids.extend(record.get("user_ids", []))
# get sfa records for all records associated with these records.
# we'll replace pl ids (person_ids) with hrns from the sfa records
# we obtain
# get the registry records
user_list, users = [], {}
user_list = dbsession.query(RegRecord).filter(RegRecord.pointer.in_(user_ids))
# create a hrns keyed on the sfa record's pointer.
# Its possible for multiple records to have the same pointer so
# the dict's value will be a list of hrns.
users = defaultdict(list)
for user in user_list:
users[user.pointer].append(user)
# get the dummy records
dummy_user_list, dummy_users = [], {}
dummy_user_list = self.shell.GetUsers({"user_ids": user_ids})
dummy_users = list_to_dict(dummy_user_list, "user_id")
# fill sfa info
for record in records:
# skip records with no pl info (top level authorities)
# if record['pointer'] == -1:
# continue
sfa_info = {}
type = record["type"]
logger.info("fill_record_sfa_info - incoming record typed %s" % type)
if type == "slice":
# all slice users are researchers
record["geni_urn"] = hrn_to_urn(record["hrn"], "slice")
record["PI"] = []
record["researcher"] = []
for user_id in record.get("user_ids", []):
hrns = [user.hrn for user in users[user_id]]
record["researcher"].extend(hrns)
elif type.startswith("authority"):
record["url"] = None
logger.info("fill_record_sfa_info - authority xherex")
elif type == "node":
sfa_info["dns"] = record.get("hostname", "")
# xxx TODO: URI, LatLong, IP, DNS
elif type == "user":
logger.info("setting user.email")
sfa_info["email"] = record.get("email", "")
sfa_info["geni_urn"] = hrn_to_urn(record["hrn"], "user")
sfa_info["geni_certificate"] = record["gid"]
# xxx TODO: PostalAddress, Phone
record.update(sfa_info)
def _getCredentialRaw(self):
"""
Get our current credential directly from the local registry.
"""
hrn = self.hrn
auth_hrn = self.auth.get_authority(hrn)
# is this a root or sub authority
if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
auth_hrn = hrn
auth_info = self.auth.get_auth_info(auth_hrn)
from sfa.storage.alchemy import dbsession
from sfa.storage.model import RegRecord
record = dbsession.query(RegRecord).filter_by(type='authority+sa', hrn=hrn).first()
if not record:
raise RecordNotFound(hrn)
type = record.type
object_gid = record.get_gid_object()
new_cred = Credential(subject = object_gid.get_subject())
new_cred.set_gid_caller(object_gid)
new_cred.set_gid_object(object_gid)
new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
r1 = determine_rights(type, hrn)
new_cred.set_privileges(r1)
new_cred.encode()
new_cred.sign()
return new_cred
def get_key_from_incoming_ip (self, api):
# verify that the callers's ip address exist in the db and is an interface
# for a node in the db
(ip, port) = api.remote_addr
interfaces = self.driver.shell.GetInterfaces({'ip': ip}, ['node_id'])
if not interfaces:
raise NonExistingRecord("no such ip %(ip)s" % locals())
nodes = self.driver.shell.GetNodes([interfaces[0]['node_id']], ['node_id', 'hostname'])
if not nodes:
raise NonExistingRecord("no such node using ip %(ip)s" % locals())
node = nodes[0]
# look up the sfa record
record=dbsession.query(RegRecord).filter_by(type='node',pointer=node['node_id']).first()
if not record:
raise RecordNotFound("node with pointer %s"%node['node_id'])
# generate a new keypair and gid
uuid = create_uuid()
pkey = Keypair(create=True)
urn = hrn_to_urn(record.hrn, record.type)
gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
gid = gid_object.save_to_string(save_parents=True)
record.gid = gid
# update the record
dbsession.commit()
# attempt the scp the key
# and gid onto the node
# this will only work for planetlab based components
(kfd, key_filename) = tempfile.mkstemp()
(gfd, gid_filename) = tempfile.mkstemp()
pkey.save_to_file(key_filename)
gid_object.save_to_file(gid_filename, save_parents=True)
host = node['hostname']
key_dest="/etc/sfa/node.key"
gid_dest="/etc/sfa/node.gid"
scp = "/usr/bin/scp"
#identity = "/etc/planetlab/root_ssh_key.rsa"
identity = "/etc/sfa/root_ssh_key"
scp_options=" -i %(identity)s " % locals()
scp_options+="-o StrictHostKeyChecking=no " % locals()
scp_key_command="%(scp)s %(scp_options)s %(key_filename)s root@%(host)s:%(key_dest)s" %\
locals()
scp_gid_command="%(scp)s %(scp_options)s %(gid_filename)s root@%(host)s:%(gid_dest)s" %\
locals()
all_commands = [scp_key_command, scp_gid_command]
for command in all_commands:
(status, output) = commands.getstatusoutput(command)
if status:
raise Exception, output
for filename in [key_filename, gid_filename]:
os.unlink(filename)
return 1
def update_relation (self, record_obj, field_key, hrns, target_type):
# locate the linked objects in our db
subject_type=record_obj.type
subject_id=record_obj.pointer
# get the 'pointer' field of all matching records
link_id_tuples = dbsession.query(RegRecord.pointer).filter_by(type=target_type).filter(RegRecord.hrn.in_(hrns)).all()
# sqlalchemy returns named tuples for columns
link_ids = [ tuple.pointer for tuple in link_id_tuples ]
self.driver.update_relation (subject_type, target_type, subject_id, link_ids)
def update_relation(self, record_obj, field_key, hrns, target_type):
# locate the linked objects in our db
subject_type = record_obj.type
subject_id = record_obj.pointer
# get the 'pointer' field of all matching records
link_id_tuples = dbsession.query(RegRecord.pointer).filter_by(
type=target_type).filter(RegRecord.hrn.in_(hrns)).all()
# sqlalchemy returns named tuples for columns
link_ids = [tuple.pointer for tuple in link_id_tuples]
self.driver.update_relation(subject_type, target_type, subject_id,
link_ids)
def fill_record_sfa_info(self, records):
def startswith(prefix, values):
return [value for value in values if value.startswith(prefix)]
# get user ids
user_ids = []
for record in records:
user_ids.extend(record.get("user_ids", []))
# get the registry records
user_list, users = [], {}
user_list = dbsession.query(RegRecord).filter(RegRecord.pointer.in_(user_ids)).all()
# create a hrns keyed on the sfa record's pointer.
# Its possible for multiple records to have the same pointer so
# the dict's value will be a list of hrns.
users = defaultdict(list)
for user in user_list:
users[user.pointer].append(user)
# get the nitos records
nitos_user_list, nitos_users = [], {}
nitos_all_users = self.convert_id(self.shell.getUsers())
nitos_user_list = [user for user in nitos_all_users if user['user_id'] in user_ids]
nitos_users = list_to_dict(nitos_user_list, 'user_id')
# fill sfa info
for record in records:
if record['pointer'] == -1:
continue
sfa_info = {}
type = record['type']
logger.info("fill_record_sfa_info - incoming record typed %s"%type)
if (type == "slice"):
# all slice users are researchers
record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
record['researcher'] = []
for user_id in record.get('user_ids', []):
hrns = [user.hrn for user in users[user_id]]
record['researcher'].extend(hrns)
elif (type == "node"):
sfa_info['dns'] = record.get("hostname", "")
# xxx TODO: URI, LatLong, IP, DNS
elif (type == "user"):
logger.info('setting user.email')
sfa_info['email'] = record.get("email", "")
sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
sfa_info['geni_certificate'] = record['gid']
# xxx TODO: PostalAddress, Phone
record.update(sfa_info)
def TestSQL(arg = None):
from sfa.storage.model import make_record, RegRecord, RegAuthority, RegUser, RegSlice, RegKey
from sfa.storage.alchemy import dbsession
from sqlalchemy.orm.collections import InstrumentedList
from sqlalchemy.orm import joinedload
solo_query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).filter_by(hrn='senslab2.avakian_slice').first()
print "\r\n \r\n =========== query_slice_list RegSlice joinedload('reg_researchers') senslab2.avakian first \r\n \t ",solo_query_slice_list.__dict__
query_slice_list = dbsession.query(RegSlice).options(joinedload('reg_researchers')).all()
print "\r\n \r\n =========== query_slice_list RegSlice joinedload('reg_researchers') ALL \r\n \t", query_slice_list[0].__dict__
return_slicerec_dictlist = []
record = query_slice_list[0]
print "\r\n \r\n =========== \r\n \t", record
tmp = record.__dict__
print "\r\n \r\n =========== \r\n \t", tmp
tmp['reg_researchers'] = tmp['reg_researchers'][0].__dict__
print "\r\n \r\n =========== \r\n \t", tmp
#del tmp['reg_researchers']['_sa_instance_state']
return_slicerec_dictlist.append(tmp)
print "\r\n \r\n =========== \r\n \t",return_slicerec_dictlist
def run (self, options):
# we don't have any options for now
self.logger.info ("OpenstackImporter.run : to do")
# create dict of all existing sfa records
existing_records = {}
existing_hrns = []
key_ids = []
for record in dbsession.query(RegRecord):
existing_records[ (record.hrn, record.type,) ] = record
existing_hrns.append(record.hrn)
tenants_dict = self.import_tenants(existing_hrns, existing_records)
users_dict, user_keys = self.import_users(existing_hrns, existing_records)
# remove stale records
system_records = [self.interface_hrn, self.root_auth, self.interface_hrn + '.slicemanager']
for (record_hrn, type) in existing_records.keys():
if record_hrn in system_records:
continue
record = existing_records[(record_hrn, type)]
if record.peer_authority:
continue
if type == 'user':
if record_hrn in users_dict:
continue
elif type in['slice', 'authority']:
if record_hrn in tenants_dict:
continue
else:
continue
record_object = existing_records[ (record_hrn, type) ]
self.logger.info("OpenstackImporter: removing %s " % record)
dbsession.delete(record_object)
dbsession.commit()
# save pub keys
self.logger.info('OpenstackImporter: saving current pub keys')
keys_filename = self.config.config_path + os.sep + 'person_keys.py'
save_keys(keys_filename, user_keys)
def Remove(self, api, xrn, origin_hrn=None):
hrn=xrn.get_hrn()
type=xrn.get_type()
request=dbsession.query(RegRecord).filter_by(hrn=hrn)
if type and type not in ['all', '*']:
request=request.filter_by(type=type)
record = request.first()
if not record:
msg="Could not find hrn %s"%hrn
if type: msg += " type=%s"%type
raise RecordNotFound(msg)
type = record.type
if type not in ['slice', 'user', 'node', 'authority'] :
raise UnknownSfaType(type)
credential = api.getCredential()
registries = api.registries
# Try to remove the object from the PLCDB of federated agg.
# This is attempted before removing the object from the local agg's PLCDB and sfa table
if hrn.startswith(api.hrn) and type in ['user', 'slice', 'authority']:
for registry in registries:
if registry not in [api.hrn]:
try:
result=registries[registry].remove_peer_object(credential, record, origin_hrn)
except:
pass
# call testbed callback first
# IIUC this is done on the local testbed TOO because of the refreshpeer link
if not self.driver.remove(record.__dict__):
logger.warning("driver.remove failed")
# delete from sfa db
dbsession.delete(record)
dbsession.commit()
return 1
def export(self, xrn, type=None, outfile=None):
"""Fetch an object's GID from the Registry"""
from sfa.storage.alchemy import dbsession
from sfa.storage.model import RegRecord
hrn = Xrn(xrn).get_hrn()
request=dbsession.query(RegRecord).filter_by(hrn=hrn)
if type: request = request.filter_by(type=type)
record=request.first()
if record:
gid = GID(string=record.gid)
else:
# check the authorities hierarchy
hierarchy = Hierarchy()
try:
auth_info = hierarchy.get_auth_info(hrn)
gid = auth_info.gid_object
except:
print "Record: %s not found" % hrn
sys.exit(1)
# save to file
if not outfile:
outfile = os.path.abspath('./%s.gid' % gid.get_hrn())
gid.save_to_file(outfile, save_parents=True)
def record_exists (self, type, hrn):
return dbsession.query(RegRecord).filter_by(hrn=hrn,type=type).count()!=0
def check_gid(self, xrn=None, type=None, all=None, verbose=None):
"""Check the correspondance between the GID and the PubKey"""
# db records
from sfa.storage.alchemy import dbsession
from sfa.storage.model import RegRecord
db_query = dbsession.query(RegRecord).filter_by(type=type)
if xrn and not all:
hrn = Xrn(xrn).get_hrn()
db_query = db_query.filter_by(hrn=hrn)
elif all and xrn:
print "Use either -a or -x <xrn>, not both !!!"
sys.exit(1)
elif not all and not xrn:
print "Use either -a or -x <xrn>, one of them is mandatory !!!"
sys.exit(1)
records = db_query.all()
if not records:
print "No Record found"
sys.exit(1)
OK = []
NOK = []
ERROR = []
NOKEY = []
for record in records:
# get the pubkey stored in SFA DB
if record.reg_keys:
db_pubkey_str = record.reg_keys[0].key
try:
db_pubkey_obj = convert_public_key(db_pubkey_str)
except:
ERROR.append(record.hrn)
continue
else:
NOKEY.append(record.hrn)
continue
# get the pubkey from the gid
gid_str = record.gid
gid_obj = GID(string = gid_str)
gid_pubkey_obj = gid_obj.get_pubkey()
# Check if gid_pubkey_obj and db_pubkey_obj are the same
check = gid_pubkey_obj.is_same(db_pubkey_obj)
if check :
OK.append(record.hrn)
else:
NOK.append(record.hrn)
if not verbose:
print "Users NOT having a PubKey: %s\n\
Users having a non RSA PubKey: %s\n\
Users having a GID/PubKey correpondence OK: %s\n\
Users having a GID/PubKey correpondence Not OK: %s\n"%(len(NOKEY), len(ERROR), len(OK), len(NOK))
else:
print "Users NOT having a PubKey: %s and are: \n%s\n\n\
Users having a non RSA PubKey: %s and are: \n%s\n\n\
Users having a GID/PubKey correpondence OK: %s and are: \n%s\n\n\
Users having a GID/PubKey correpondence NOT OK: %s and are: \n%s\n\n"%(len(NOKEY),NOKEY, len(ERROR), ERROR, len(OK), OK, len(NOK), NOK)
def run (self, options):
config = Config ()
interface_hrn = config.SFA_INTERFACE_HRN
root_auth = config.SFA_REGISTRY_ROOT_AUTH
shell = DummyShell (config)
######## retrieve all existing SFA objects
all_records = dbsession.query(RegRecord).all()
# create hash by (type,hrn)
# we essentially use this to know if a given record is already known to SFA
self.records_by_type_hrn = \
dict ( [ ( (record.type, record.hrn) , record ) for record in all_records ] )
# create hash by (type,pointer)
self.records_by_type_pointer = \
dict ( [ ( (record.type, record.pointer) , record ) for record in all_records
if record.pointer != -1] )
# initialize record.stale to True by default, then mark stale=False on the ones that are in use
for record in all_records: record.stale=True
######## retrieve Dummy TB data
# Get all plc sites
# retrieve only required stuf
sites = [shell.GetTestbedInfo()]
# create a hash of sites by login_base
# sites_by_login_base = dict ( [ ( site['login_base'], site ) for site in sites ] )
# Get all dummy TB users
users = shell.GetUsers()
# create a hash of users by user_id
users_by_id = dict ( [ ( user['user_id'], user) for user in users ] )
# Get all dummy TB public keys
keys = []
for user in users:
if 'keys' in user:
keys.extend(user['keys'])
# create a dict user_id -> [ keys ]
keys_by_person_id = {}
for user in users:
if 'keys' in user:
keys_by_person_id[user['user_id']] = user['keys']
# Get all dummy TB nodes
nodes = shell.GetNodes()
# create hash by node_id
nodes_by_id = dict ( [ ( node['node_id'], node, ) for node in nodes ] )
# Get all dummy TB slices
slices = shell.GetSlices()
# create hash by slice_id
slices_by_id = dict ( [ (slice['slice_id'], slice ) for slice in slices ] )
# start importing
for site in sites:
site_hrn = _get_site_hrn(interface_hrn, site)
# import if hrn is not in list of existing hrns or if the hrn exists
# but its not a site record
site_record=self.locate_by_type_hrn ('authority', site_hrn)
if not site_record:
try:
urn = hrn_to_urn(site_hrn, 'authority')
if not self.auth_hierarchy.auth_exists(urn):
self.auth_hierarchy.create_auth(urn)
auth_info = self.auth_hierarchy.get_auth_info(urn)
site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(),
pointer= -1,
authority=get_authority(site_hrn))
site_record.just_created()
dbsession.add(site_record)
dbsession.commit()
self.logger.info("DummyImporter: imported authority (site) : %s" % site_record)
self.remember_record (site_record)
except:
# if the site import fails then there is no point in trying to import the
# site's child records (node, slices, persons), so skip them.
self.logger.log_exc("DummyImporter: failed to import site. Skipping child records")
continue
else:
# xxx update the record ...
pass
site_record.stale=False
# import node records
for node in nodes:
site_auth = get_authority(site_hrn)
site_name = site['name']
node_hrn = hostname_to_hrn(site_auth, site_name, node['hostname'])
# xxx this sounds suspicious
if len(node_hrn) > 64: node_hrn = node_hrn[:64]
node_record = self.locate_by_type_hrn ( 'node', node_hrn )
if not node_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(node_hrn, 'node')
node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
node_record = RegNode (hrn=node_hrn, gid=node_gid,
pointer =node['node_id'],
authority=get_authority(node_hrn))
node_record.just_created()
dbsession.add(node_record)
dbsession.commit()
self.logger.info("DummyImporter: imported node: %s" % node_record)
self.remember_record (node_record)
except:
self.logger.log_exc("DummyImporter: failed to import node")
else:
# xxx update the record ...
pass
node_record.stale=False
site_pis=[]
# import users
for user in users:
user_hrn = email_to_hrn(site_hrn, user['email'])
# xxx suspicious again
if len(user_hrn) > 64: user_hrn = user_hrn[:64]
user_urn = hrn_to_urn(user_hrn, 'user')
user_record = self.locate_by_type_hrn ( 'user', user_hrn)
# return a tuple pubkey (a dummy TB key object) and pkey (a Keypair object)
def init_user_key (user):
pubkey = None
pkey = None
if user['keys']:
# randomly pick first key in set
for key in user['keys']:
pubkey = key
try:
pkey = convert_public_key(pubkey)
break
except:
continue
if not pkey:
self.logger.warn('DummyImporter: unable to convert public key for %s' % user_hrn)
pkey = Keypair(create=True)
else:
# the user has no keys. Creating a random keypair for the user's gid
self.logger.warn("DummyImporter: user %s does not have a NITOS public key"%user_hrn)
pkey = Keypair(create=True)
return (pubkey, pkey)
# new user
try:
if not user_record:
(pubkey,pkey) = init_user_key (user)
user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey)
user_gid.set_email(user['email'])
user_record = RegUser (hrn=user_hrn, gid=user_gid,
pointer=user['user_id'],
authority=get_authority(user_hrn),
email=user['email'])
if pubkey:
user_record.reg_keys=[RegKey (pubkey)]
else:
self.logger.warning("No key found for user %s"%user_record)
user_record.just_created()
dbsession.add (user_record)
dbsession.commit()
self.logger.info("DummyImporter: imported person: %s" % user_record)
self.remember_record ( user_record )
else:
# update the record ?
# if user's primary key has changed then we need to update the
# users gid by forcing an update here
sfa_keys = user_record.reg_keys
def key_in_list (key,sfa_keys):
for reg_key in sfa_keys:
if reg_key.key==key: return True
return False
# is there a new key in Dummy TB ?
new_keys=False
for key in user['keys']:
if not key_in_list (key,sfa_keys):
new_keys = True
if new_keys:
(pubkey,pkey) = init_user_key (user)
user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey)
if not pubkey:
user_record.reg_keys=[]
else:
user_record.reg_keys=[ RegKey (pubkey)]
self.logger.info("DummyImporter: updated person: %s" % user_record)
user_record.email = user['email']
dbsession.commit()
user_record.stale=False
except:
self.logger.log_exc("DummyImporter: failed to import user %d %s"%(user['user_id'],user['email']))
# import slices
for slice in slices:
slice_hrn = slicename_to_hrn(site_hrn, slice['slice_name'])
slice_record = self.locate_by_type_hrn ('slice', slice_hrn)
if not slice_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(slice_hrn, 'slice')
slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
pointer=slice['slice_id'],
authority=get_authority(slice_hrn))
slice_record.just_created()
dbsession.add(slice_record)
dbsession.commit()
self.logger.info("DummyImporter: imported slice: %s" % slice_record)
self.remember_record ( slice_record )
except:
self.logger.log_exc("DummyImporter: failed to import slice")
else:
# xxx update the record ...
self.logger.warning ("Slice update not yet implemented")
pass
# record current users affiliated with the slice
slice_record.reg_researchers = \
[ self.locate_by_type_pointer ('user',user_id) for user_id in slice['user_ids'] ]
dbsession.commit()
slice_record.stale=False
### remove stale records
# special records must be preserved
system_hrns = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
for record in all_records:
if record.hrn in system_hrns:
record.stale=False
if record.peer_authority:
record.stale=False
for record in all_records:
try: stale=record.stale
except:
stale=True
self.logger.warning("stale not found with %s"%record)
if stale:
self.logger.info("DummyImporter: deleting stale record: %s" % record)
dbsession.delete(record)
dbsession.commit()
def run (self, options):
config = Config ()
interface_hrn = config.SFA_INTERFACE_HRN
root_auth = config.SFA_REGISTRY_ROOT_AUTH
shell = PlShell (config)
######## retrieve all existing SFA objects
all_records = dbsession.query(RegRecord).all()
# create hash by (type,hrn)
# we essentially use this to know if a given record is already known to SFA
self.records_by_type_hrn = \
dict ( [ ( (record.type, record.hrn) , record ) for record in all_records ] )
# create hash by (type,pointer)
self.records_by_type_pointer = \
dict ( [ ( (record.type, record.pointer) , record ) for record in all_records
if record.pointer != -1] )
# initialize record.stale to True by default, then mark stale=False on the ones that are in use
for record in all_records: record.stale=True
######## retrieve PLC data
# Get all plc sites
# retrieve only required stuf
sites = shell.GetSites({'peer_id': None, 'enabled' : True},
['site_id','login_base','node_ids','slice_ids','person_ids',])
# create a hash of sites by login_base
# sites_by_login_base = dict ( [ ( site['login_base'], site ) for site in sites ] )
# Get all plc users
persons = shell.GetPersons({'peer_id': None, 'enabled': True},
['person_id', 'email', 'key_ids', 'site_ids', 'role_ids'])
# create a hash of persons by person_id
persons_by_id = dict ( [ ( person['person_id'], person) for person in persons ] )
# also gather non-enabled user accounts so as to issue relevant warnings
disabled_persons = shell.GetPersons({'peer_id': None, 'enabled': False}, ['person_id'])
disabled_person_ids = [ person['person_id'] for person in disabled_persons ]
# Get all plc public keys
# accumulate key ids for keys retrieval
key_ids = []
for person in persons:
key_ids.extend(person['key_ids'])
keys = shell.GetKeys( {'peer_id': None, 'key_id': key_ids,
'key_type': 'ssh'} )
# create a hash of keys by key_id
keys_by_id = dict ( [ ( key['key_id'], key ) for key in keys ] )
# create a dict person_id -> [ (plc)keys ]
keys_by_person_id = {}
for person in persons:
pubkeys = []
for key_id in person['key_ids']:
# by construction all the keys we fetched are ssh keys
# so gpg keys won't be in there
try:
key = keys_by_id[key_id]
pubkeys.append(key)
except:
self.logger.warning("Could not spot key %d - probably non-ssh"%key_id)
keys_by_person_id[person['person_id']] = pubkeys
# Get all plc nodes
nodes = shell.GetNodes( {'peer_id': None}, ['node_id', 'hostname', 'site_id'])
# create hash by node_id
nodes_by_id = dict ( [ ( node['node_id'], node, ) for node in nodes ] )
# Get all plc slices
slices = shell.GetSlices( {'peer_id': None}, ['slice_id', 'name', 'person_ids'])
# create hash by slice_id
slices_by_id = dict ( [ (slice['slice_id'], slice ) for slice in slices ] )
# isolate special vini case in separate method
self.create_special_vini_record (interface_hrn)
# start importing
for site in sites:
site_hrn = _get_site_hrn(interface_hrn, site)
# import if hrn is not in list of existing hrns or if the hrn exists
# but its not a site record
site_record=self.locate_by_type_hrn ('authority', site_hrn)
if not site_record:
try:
urn = hrn_to_urn(site_hrn, 'authority')
if not self.auth_hierarchy.auth_exists(urn):
self.auth_hierarchy.create_auth(urn)
auth_info = self.auth_hierarchy.get_auth_info(urn)
site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(),
pointer=site['site_id'],
authority=get_authority(site_hrn))
site_record.just_created()
dbsession.add(site_record)
dbsession.commit()
self.logger.info("PlImporter: imported authority (site) : %s" % site_record)
self.remember_record (site_record)
except:
# if the site import fails then there is no point in trying to import the
# site's child records (node, slices, persons), so skip them.
self.logger.log_exc("PlImporter: failed to import site %s. Skipping child records"%site_hrn)
continue
else:
# xxx update the record ...
pass
site_record.stale=False
# import node records
for node_id in site['node_ids']:
try:
node = nodes_by_id[node_id]
except:
self.logger.warning ("PlImporter: cannot find node_id %s - ignored"%node_id)
continue
site_auth = get_authority(site_hrn)
site_name = site['login_base']
node_hrn = hostname_to_hrn(site_auth, site_name, node['hostname'])
# xxx this sounds suspicious
if len(node_hrn) > 64: node_hrn = node_hrn[:64]
node_record = self.locate_by_type_hrn ( 'node', node_hrn )
if not node_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(node_hrn, 'node')
node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
node_record = RegNode (hrn=node_hrn, gid=node_gid,
pointer =node['node_id'],
authority=get_authority(node_hrn))
node_record.just_created()
dbsession.add(node_record)
dbsession.commit()
self.logger.info("PlImporter: imported node: %s" % node_record)
self.remember_record (node_record)
except:
self.logger.log_exc("PlImporter: failed to import node %s"%node_hrn)
continue
else:
# xxx update the record ...
pass
node_record.stale=False
site_pis=[]
# import persons
for person_id in site['person_ids']:
proceed=False
if person_id in persons_by_id:
person=persons_by_id[person_id]
proceed=True
elif person_id in disabled_person_ids:
pass
else:
self.logger.warning ("PlImporter: cannot locate person_id %s in site %s - ignored"%(person_id,site_hrn))
# make sure to NOT run this if anything is wrong
if not proceed: continue
person_hrn = email_to_hrn(site_hrn, person['email'])
# xxx suspicious again
if len(person_hrn) > 64: person_hrn = person_hrn[:64]
person_urn = hrn_to_urn(person_hrn, 'user')
user_record = self.locate_by_type_hrn ( 'user', person_hrn)
# return a tuple pubkey (a plc key object) and pkey (a Keypair object)
def init_person_key (person, plc_keys):
pubkey=None
if person['key_ids']:
# randomly pick first key in set
pubkey = plc_keys[0]
try:
pkey = convert_public_key(pubkey['key'])
except:
self.logger.warn('PlImporter: unable to convert public key for %s' % person_hrn)
pkey = Keypair(create=True)
else:
# the user has no keys. Creating a random keypair for the user's gid
self.logger.warn("PlImporter: person %s does not have a PL public key"%person_hrn)
pkey = Keypair(create=True)
return (pubkey, pkey)
# new person
try:
plc_keys = keys_by_person_id.get(person['person_id'],[])
if not user_record:
(pubkey,pkey) = init_person_key (person, plc_keys )
person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey)
person_gid.set_email(person['email'])
user_record = RegUser (hrn=person_hrn, gid=person_gid,
pointer=person['person_id'],
authority=get_authority(person_hrn),
email=person['email'])
if pubkey:
user_record.reg_keys=[RegKey (pubkey['key'], pubkey['key_id'])]
else:
self.logger.warning("No key found for user %s"%user_record)
user_record.just_created()
dbsession.add (user_record)
dbsession.commit()
self.logger.info("PlImporter: imported person: %s" % user_record)
self.remember_record ( user_record )
else:
# update the record ?
#
# if a user key has changed then we need to update the
# users gid by forcing an update here
#
# right now, SFA only has *one* key attached to a user, and this is
# the key that the GID was made with
# so the logic here is, we consider that things are OK (unchanged) if
# all the SFA keys are present as PLC keys
# otherwise we trigger the creation of a new gid from *some* plc key
# and record this on the SFA side
# it would make sense to add a feature in PLC so that one could pick a 'primary'
# key but this is not available on the myplc side for now
# = or = it would be much better to support several keys in SFA but that
# does not seem doable without a major overhaul in the data model as
# a GID is attached to a hrn, but it's also linked to a key, so...
# NOTE: with this logic, the first key entered in PLC remains the one
# current in SFA until it is removed from PLC
sfa_keys = user_record.reg_keys
def sfa_key_in_list (sfa_key,plc_keys):
for plc_key in plc_keys:
if plc_key['key']==sfa_key.key:
return True
return False
# are all the SFA keys known to PLC ?
new_keys=False
if not sfa_keys and plc_keys:
new_keys=True
else:
for sfa_key in sfa_keys:
if not sfa_key_in_list (sfa_key,plc_keys):
new_keys = True
if new_keys:
(pubkey,pkey) = init_person_key (person, plc_keys)
person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey)
person_gid.set_email(person['email'])
if not pubkey:
user_record.reg_keys=[]
else:
user_record.reg_keys=[ RegKey (pubkey['key'], pubkey['key_id'])]
user_record.gid = person_gid
user_record.just_updated()
self.logger.info("PlImporter: updated person: %s" % user_record)
user_record.email = person['email']
dbsession.commit()
user_record.stale=False
# accumulate PIs - PLCAPI has a limitation that when someone has PI role
# this is valid for all sites she is in..
# PI is coded with role_id==20
if 20 in person['role_ids']:
site_pis.append (user_record)
except:
self.logger.log_exc("PlImporter: failed to import person %d %s"%(person['person_id'],person['email']))
# maintain the list of PIs for a given site
# for the record, Jordan had proposed the following addition as a welcome hotfix to a previous version:
# site_pis = list(set(site_pis))
# this was likely due to a bug in the above logic, that had to do with disabled persons
# being improperly handled, and where the whole loop on persons
# could be performed twice with the same person...
# so hopefully we do not need to eliminate duplicates explicitly here anymore
site_record.reg_pis = site_pis
dbsession.commit()
# import slices
for slice_id in site['slice_ids']:
try:
slice = slices_by_id[slice_id]
except:
self.logger.warning ("PlImporter: cannot locate slice_id %s - ignored"%slice_id)
slice_hrn = slicename_to_hrn(interface_hrn, slice['name'])
slice_record = self.locate_by_type_hrn ('slice', slice_hrn)
if not slice_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(slice_hrn, 'slice')
slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
pointer=slice['slice_id'],
authority=get_authority(slice_hrn))
slice_record.just_created()
dbsession.add(slice_record)
dbsession.commit()
self.logger.info("PlImporter: imported slice: %s" % slice_record)
self.remember_record ( slice_record )
except:
self.logger.log_exc("PlImporter: failed to import slice %s (%s)"%(slice_hrn,slice['name']))
else:
# xxx update the record ...
# given that we record the current set of users anyways, there does not seem to be much left to do here
# self.logger.warning ("Slice update not yet implemented on slice %s (%s)"%(slice_hrn,slice['name']))
pass
# record current users affiliated with the slice
slice_record.reg_researchers = \
[ self.locate_by_type_pointer ('user',user_id) for user_id in slice['person_ids'] ]
dbsession.commit()
slice_record.stale=False
### remove stale records
# special records must be preserved
system_hrns = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
for record in all_records:
if record.hrn in system_hrns:
record.stale=False
if record.peer_authority:
record.stale=False
if ".vini" in interface_hrn and interface_hrn.endswith('vini') and \
record.hrn.endswith("internet2"):
record.stale=False
for record in all_records:
try: stale=record.stale
except:
stale=True
self.logger.warning("stale not found with %s"%record)
if stale:
self.logger.info("PlImporter: deleting stale record: %s" % record)
dbsession.delete(record)
dbsession.commit()
def GetCredential(self, api, xrn, type, caller_xrn = None):
# convert xrn to hrn
if type:
hrn = urn_to_hrn(xrn)[0]
else:
hrn, type = urn_to_hrn(xrn)
# Is this a root or sub authority
auth_hrn = api.auth.get_authority(hrn)
if not auth_hrn or hrn == api.config.SFA_INTERFACE_HRN:
auth_hrn = hrn
auth_info = api.auth.get_auth_info(auth_hrn)
# get record info
filter = {'hrn': hrn}
if type:
filter['type'] = type
record=dbsession.query(RegRecord).filter_by(**filter).first()
if not record:
raise RecordNotFound("hrn=%s, type=%s"%(hrn,type))
# verify_cancreate_credential requires that the member lists
# (researchers, pis, etc) be filled in
logger.debug("get credential before augment dict, keys=%s"%record.__dict__.keys())
self.driver.augment_records_with_testbed_info (record.__dict__)
logger.debug("get credential after augment dict, keys=%s"%record.__dict__.keys())
if not self.driver.is_enabled (record.__dict__):
raise AccountNotEnabled(": PlanetLab account %s is not enabled. Please contact your site PI" %(record.email))
# get the callers gid
# if caller_xrn is not specified assume the caller is the record
# object itself.
if not caller_xrn:
caller_hrn = hrn
caller_gid = record.get_gid_object()
else:
caller_hrn, caller_type = urn_to_hrn(caller_xrn)
caller_filter = {'hrn': caller_hrn}
if caller_type:
caller_filter['type'] = caller_type
caller_record = dbsession.query(RegRecord).filter_by(**caller_filter).first()
if not caller_record:
raise RecordNotFound("Unable to associated caller (hrn=%s, type=%s) with credential for (hrn: %s, type: %s)"%(caller_hrn, caller_type, hrn, type))
caller_gid = GID(string=caller_record.gid)
object_hrn = record.get_gid_object().get_hrn()
rights = api.auth.determine_user_rights(caller_hrn, record.todict())
# make sure caller has rights to this object
if rights.is_empty():
raise PermissionError(caller_hrn + " has no rights to " + record.hrn)
object_gid = GID(string=record.gid)
new_cred = Credential(subject = object_gid.get_subject())
new_cred.set_gid_caller(caller_gid)
new_cred.set_gid_object(object_gid)
new_cred.set_issuer_keys(auth_info.get_privkey_filename(), auth_info.get_gid_filename())
#new_cred.set_pubkey(object_gid.get_pubkey())
new_cred.set_privileges(rights)
new_cred.get_privileges().delegate_all_privileges(True)
if hasattr(record,'expires'):
date = utcparse(record.expires)
expires = datetime_to_epoch(date)
new_cred.set_expiration(int(expires))
auth_kind = "authority,ma,sa"
# Parent not necessary, verify with certs
#new_cred.set_parent(api.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))
new_cred.encode()
new_cred.sign()
return new_cred.save_to_string(save_parents=True)
def run (self, options):
config = Config()
slabdriver = SlabDriver(config)
#Create special slice table for senslab
if not slabdriver.db.exists('slab_xp'):
slabdriver.db.createtable()
self.logger.info ("SlabImporter.run: slab_xp table created ")
#retrieve all existing SFA objects
all_records = dbsession.query(RegRecord).all()
#create hash by (type,hrn)
#used to know if a given record is already known to SFA
self.records_by_type_hrn = \
dict ( [ ( (record.type,record.hrn) , record ) for record in all_records ] )
print>>sys.stderr,"\r\n SLABIMPORT \t all_records[0] %s all_records[0].email %s \r\n" %(all_records[0].type, all_records[0])
self.users_rec_by_email = \
dict ( [ (record.email, record) for record in all_records if record.type == 'user' ] )
# create hash by (type,pointer)
self.records_by_type_pointer = \
dict ( [ ( (str(record.type),record.pointer) , record ) for record in all_records if record.pointer != -1] )
# initialize record.stale to True by default, then mark stale=False on the ones that are in use
for record in all_records:
record.stale=True
nodes_listdict = slabdriver.GetNodes()
nodes_by_id = dict([(node['node_id'],node) for node in nodes_listdict])
sites_listdict = slabdriver.GetSites()
ldap_person_listdict = slabdriver.GetPersons()
print>>sys.stderr,"\r\n SLABIMPORT \t ldap_person_listdict %s \r\n" %(ldap_person_listdict)
slices_listdict = slabdriver.GetSlices()
try:
slices_by_userid = dict ( [ (one_slice['reg_researchers']['record_id'], one_slice ) for one_slice in slices_listdict ] )
except TypeError:
self.logger.log_exc("SlabImporter: failed to create list of slices by user id.")
pass
for site in sites_listdict:
site_hrn = _get_site_hrn(site)
site_record = self.find_record_by_type_hrn ('authority', site_hrn)
if not site_record:
try:
urn = hrn_to_urn(site_hrn, 'authority')
if not self.auth_hierarchy.auth_exists(urn):
self.auth_hierarchy.create_auth(urn)
auth_info = self.auth_hierarchy.get_auth_info(urn)
site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(),
pointer='-1',
authority=get_authority(site_hrn))
site_record.just_created()
dbsession.add(site_record)
dbsession.commit()
self.logger.info("SlabImporter: imported authority (site) : %s" % site_record)
self.update_just_added_records_dict(site_record)
except SQLAlchemyError:
# if the site import fails then there is no point in trying to import the
# site's child records (node, slices, persons), so skip them.
self.logger.log_exc("SlabImporter: failed to import site. Skipping child records")
continue
else:
# xxx update the record ...
pass
site_record.stale=False
# import node records in site
for node_id in site['node_ids']:
try:
node = nodes_by_id[node_id]
except:
self.logger.warning ("SlabImporter: cannot find node_id %s - ignored"%node_id)
continue
site_auth = get_authority(site_hrn)
site_name = site['name']
escaped_hrn = self.hostname_to_hrn_escaped(slabdriver.root_auth, node['hostname'])
print>>sys.stderr, "\r\n \r\n SLABIMPORTER node %s " %(node)
hrn = node['hrn']
# xxx this sounds suspicious
if len(hrn) > 64: hrn = hrn[:64]
node_record = self.find_record_by_type_hrn( 'node', hrn )
if not node_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(escaped_hrn, 'node')
node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
def slab_get_authority(hrn):
return hrn.split(".")[0]
node_record = RegNode (hrn=hrn, gid=node_gid,
pointer = '-1',
authority=slab_get_authority(hrn))
node_record.just_created()
dbsession.add(node_record)
dbsession.commit()
#self.logger.info("SlabImporter: imported node: %s" % node_record)
self.update_just_added_records_dict(node_record)
except:
self.logger.log_exc("SlabImporter: failed to import node")
else:
# xxx update the record ...
pass
node_record.stale=False
# import persons
for person in ldap_person_listdict :
print>>sys.stderr,"SlabImporter: person: %s" %(person['hrn'])
if 'ssh-rsa' not in person['pkey']:
#people with invalid ssh key (ssh-dss, empty, bullshit keys...)
#won't be imported
continue
person_hrn = person['hrn']
slice_hrn = self.slicename_to_hrn(person['hrn'])
# xxx suspicious again
if len(person_hrn) > 64: person_hrn = person_hrn[:64]
person_urn = hrn_to_urn(person_hrn, 'user')
print>>sys.stderr," \r\n SlabImporter: HEYYYYYYYYYY" , self.users_rec_by_email
#Check if user using person['email'] form LDAP is already registered
#in SFA. One email = one person. Inb this case, do not create another
#record for this person
#person_hrn returned by GetPErson based on senslab root auth + uid ldap
user_record = self.find_record_by_type_hrn('user', person_hrn)
if not user_record and person['email'] in self.users_rec_by_email:
user_record = self.users_rec_by_email[person['email']]
person_hrn = user_record.hrn
person_urn = hrn_to_urn(person_hrn, 'user')
slice_record = self.find_record_by_type_hrn ('slice', slice_hrn)
# return a tuple pubkey (a plc key object) and pkey (a Keypair object)
def init_person_key (person, slab_key):
pubkey = None
if person['pkey']:
# randomly pick first key in set
pubkey = slab_key
try:
pkey = convert_public_key(pubkey)
except TypeError:
#key not good. create another pkey
self.logger.warn('SlabImporter: \
unable to convert public \
key for %s' % person_hrn)
pkey = Keypair(create=True)
else:
# the user has no keys. Creating a random keypair for the user's gid
self.logger.warn("SlabImporter: person %s does not have a public key"%person_hrn)
pkey = Keypair(create=True)
return (pubkey, pkey)
try:
slab_key = person['pkey']
# new person
if not user_record:
(pubkey,pkey) = init_person_key (person, slab_key )
if pubkey is not None and pkey is not None :
person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey)
if person['email']:
print>>sys.stderr, "\r\n \r\n SLAB IMPORTER PERSON EMAIL OK email %s " %(person['email'])
person_gid.set_email(person['email'])
user_record = RegUser (hrn=person_hrn, gid=person_gid,
pointer='-1',
authority=get_authority(person_hrn),
email=person['email'])
else:
user_record = RegUser (hrn=person_hrn, gid=person_gid,
pointer='-1',
authority=get_authority(person_hrn))
if pubkey:
user_record.reg_keys = [RegKey (pubkey)]
else:
self.logger.warning("No key found for user %s"%user_record)
user_record.just_created()
dbsession.add (user_record)
dbsession.commit()
self.logger.info("SlabImporter: imported person: %s" % user_record)
self.update_just_added_records_dict( user_record )
else:
# update the record ?
# if user's primary key has changed then we need to update the
# users gid by forcing an update here
sfa_keys = user_record.reg_keys
new_key=False
if slab_key is not sfa_keys :
new_key = True
if new_key:
print>>sys.stderr,"SlabImporter: \t \t USER UPDATE person: %s" %(person['hrn'])
(pubkey,pkey) = init_person_key (person, slab_key)
person_gid = self.auth_hierarchy.create_gid(person_urn, create_uuid(), pkey)
if not pubkey:
user_record.reg_keys=[]
else:
user_record.reg_keys=[ RegKey (pubkey)]
self.logger.info("SlabImporter: updated person: %s" % user_record)
if person['email']:
user_record.email = person['email']
dbsession.commit()
user_record.stale = False
except:
self.logger.log_exc("SlabImporter: failed to import person %s"%(person) )
try:
slice = slices_by_userid[user_record.record_id]
except:
self.logger.warning ("SlabImporter: cannot locate slices_by_userid[user_record.record_id] %s - ignored"%user_record)
if not slice_record :
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(slice_hrn, 'slice')
slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
pointer='-1',
authority=get_authority(slice_hrn))
slice_record.just_created()
dbsession.add(slice_record)
dbsession.commit()
#Serial id created after commit
#Get it
sl_rec = dbsession.query(RegSlice).filter(RegSlice.hrn.match(slice_hrn)).all()
#slab_slice = SenslabXP( slice_hrn = slice_hrn, record_id_slice=sl_rec[0].record_id, record_id_user= user_record.record_id)
#print>>sys.stderr, "\r\n \r\n SLAB IMPORTER SLICE IMPORT NOTslice_record %s \r\n slab_slice %s" %(sl_rec,slab_slice)
#slab_dbsession.add(slab_slice)
#slab_dbsession.commit()
#self.logger.info("SlabImporter: imported slice: %s" % slice_record)
self.update_just_added_records_dict ( slice_record )
except:
self.logger.log_exc("SlabImporter: failed to import slice")
#No slice update upon import in senslab
else:
# xxx update the record ...
self.logger.warning ("Slice update not yet implemented")
pass
# record current users affiliated with the slice
slice_record.reg_researchers = [user_record]
dbsession.commit()
slice_record.stale=False
### remove stale records
# special records must be preserved
system_hrns = [slabdriver.hrn, slabdriver.root_auth, slabdriver.hrn+ '.slicemanager']
for record in all_records:
if record.hrn in system_hrns:
record.stale=False
if record.peer_authority:
record.stale=False
for record in all_records:
if record.type == 'user':
print>>sys.stderr,"SlabImporter: stale records: hrn %s %s" %(record.hrn,record.stale)
try:
stale=record.stale
except:
stale=True
self.logger.warning("stale not found with %s"%record)
if stale:
self.logger.info("SlabImporter: deleting stale record: %s" % record)
#if record.type == 'user':
#rec = slab_dbsession.query(SenslabXP).filter_by(record_id_user = record.record_id).first()
#slab_dbsession.delete(rec)
#slab_dbsession.commit()
dbsession.delete(record)
dbsession.commit()
def Register(self, api, record_dict):
hrn, type = record_dict['hrn'], record_dict['type']
urn = hrn_to_urn(hrn,type)
# validate the type
if type not in ['authority', 'slice', 'node', 'user']:
raise UnknownSfaType(type)
# check if record_dict already exists
existing_records = dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).all()
if existing_records:
raise ExistingRecord(hrn)
assert ('type' in record_dict)
# returns the right type of RegRecord according to type in record
record = make_record(dict=record_dict)
record.just_created()
record.authority = get_authority(record.hrn)
auth_info = api.auth.get_auth_info(record.authority)
pub_key = None
# make sure record has a gid
if not record.gid:
uuid = create_uuid()
pkey = Keypair(create=True)
if getattr(record,'keys',None):
pub_key=record.keys
# use only first key in record
if isinstance(record.keys, types.ListType):
pub_key = record.keys[0]
pkey = convert_public_key(pub_key)
gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
gid = gid_object.save_to_string(save_parents=True)
record.gid = gid
if isinstance (record, RegAuthority):
# update the tree
if not api.auth.hierarchy.auth_exists(hrn):
api.auth.hierarchy.create_auth(hrn_to_urn(hrn,'authority'))
# get the GID from the newly created authority
auth_info = api.auth.get_auth_info(hrn)
gid = auth_info.get_gid_object()
record.gid=gid.save_to_string(save_parents=True)
# locate objects for relationships
pi_hrns = getattr(record,'pi',None)
if pi_hrns is not None: record.update_pis (pi_hrns)
elif isinstance (record, RegSlice):
researcher_hrns = getattr(record,'researcher',None)
if researcher_hrns is not None: record.update_researchers (researcher_hrns)
elif isinstance (record, RegUser):
# create RegKey objects for incoming keys
if hasattr(record,'keys'):
logger.debug ("creating %d keys for user %s"%(len(record.keys),record.hrn))
record.reg_keys = [ RegKey (key) for key in record.keys ]
# update testbed-specific data if needed
pointer = self.driver.register (record.__dict__, hrn, pub_key)
record.pointer=pointer
dbsession.add(record)
dbsession.commit()
# update membership for researchers, pis, owners, operators
self.update_driver_relations (record, record)
return record.get_gid_object().save_to_string(save_parents=True)
def GetCredential(self, api, xrn, type, caller_xrn=None):
# convert xrn to hrn
if type:
hrn = urn_to_hrn(xrn)[0]
else:
hrn, type = urn_to_hrn(xrn)
# Is this a root or sub authority
auth_hrn = api.auth.get_authority(hrn)
if not auth_hrn or hrn == api.config.SFA_INTERFACE_HRN:
auth_hrn = hrn
auth_info = api.auth.get_auth_info(auth_hrn)
# get record info
filter = {'hrn': hrn}
if type:
filter['type'] = type
record = dbsession.query(RegRecord).filter_by(**filter).first()
if not record:
raise RecordNotFound("hrn=%s, type=%s" % (hrn, type))
# verify_cancreate_credential requires that the member lists
# (researchers, pis, etc) be filled in
logger.debug("get credential before augment dict, keys=%s" %
record.__dict__.keys())
self.driver.augment_records_with_testbed_info(record.__dict__)
logger.debug("get credential after augment dict, keys=%s" %
record.__dict__.keys())
if not self.driver.is_enabled(record.__dict__):
raise AccountNotEnabled(
": PlanetLab account %s is not enabled. Please contact your site PI"
% (record.email))
# get the callers gid
# if caller_xrn is not specified assume the caller is the record
# object itself.
if not caller_xrn:
caller_hrn = hrn
caller_gid = record.get_gid_object()
else:
caller_hrn, caller_type = urn_to_hrn(caller_xrn)
caller_filter = {'hrn': caller_hrn}
if caller_type:
caller_filter['type'] = caller_type
caller_record = dbsession.query(RegRecord).filter_by(
**caller_filter).first()
if not caller_record:
raise RecordNotFound(
"Unable to associated caller (hrn=%s, type=%s) with credential for (hrn: %s, type: %s)"
% (caller_hrn, caller_type, hrn, type))
caller_gid = GID(string=caller_record.gid)
object_hrn = record.get_gid_object().get_hrn()
rights = api.auth.determine_user_rights(caller_hrn, record.todict())
# make sure caller has rights to this object
if rights.is_empty():
raise PermissionError(caller_hrn + " has no rights to " +
record.hrn)
object_gid = GID(string=record.gid)
new_cred = Credential(subject=object_gid.get_subject())
new_cred.set_gid_caller(caller_gid)
new_cred.set_gid_object(object_gid)
new_cred.set_issuer_keys(auth_info.get_privkey_filename(),
auth_info.get_gid_filename())
#new_cred.set_pubkey(object_gid.get_pubkey())
new_cred.set_privileges(rights)
new_cred.get_privileges().delegate_all_privileges(True)
if hasattr(record, 'expires'):
date = utcparse(record.expires)
expires = datetime_to_epoch(date)
new_cred.set_expiration(int(expires))
auth_kind = "authority,ma,sa"
# Parent not necessary, verify with certs
#new_cred.set_parent(api.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))
new_cred.encode()
new_cred.sign()
return new_cred.save_to_string(save_parents=True)
def Update(self, api, record_dict):
assert ('type' in record_dict)
new_record=make_record(dict=record_dict)
(type,hrn) = (new_record.type, new_record.hrn)
# make sure the record exists
record = dbsession.query(RegRecord).filter_by(type=type,hrn=hrn).first()
if not record:
raise RecordNotFound("hrn=%s, type=%s"%(hrn,type))
record.just_updated()
# Use the pointer from the existing record, not the one that the user
# gave us. This prevents the user from inserting a forged pointer
pointer = record.pointer
# is there a change in keys ?
new_key=None
if type=='user':
if getattr(new_record,'keys',None):
new_key=new_record.keys
if isinstance (new_key,types.ListType):
new_key=new_key[0]
# take new_key into account
if new_key:
# update the openssl key and gid
pkey = convert_public_key(new_key)
uuid = create_uuid()
urn = hrn_to_urn(hrn,type)
gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
gid = gid_object.save_to_string(save_parents=True)
# xxx should do side effects from new_record to record
# not too sure how to do that
# not too big a deal with planetlab as the driver is authoritative, but...
# update native relations
if isinstance (record, RegSlice):
researcher_hrns = getattr(new_record,'researcher',None)
if researcher_hrns is not None: record.update_researchers (researcher_hrns)
elif isinstance (record, RegAuthority):
pi_hrns = getattr(new_record,'pi',None)
if pi_hrns is not None: record.update_pis (pi_hrns)
# update the PLC information that was specified with the record
# xxx oddly enough, without this useless statement,
# record.__dict__ as received by the driver seems to be off
# anyway the driver should receive an object
# (and then extract __dict__ itself if needed)
print "DO NOT REMOVE ME before driver.update, record=%s"%record
new_key_pointer = -1
try:
(pointer, new_key_pointer) = self.driver.update (record.__dict__, new_record.__dict__, hrn, new_key)
except:
pass
if new_key and new_key_pointer:
record.reg_keys=[ RegKey (new_key, new_key_pointer)]
record.gid = gid
dbsession.commit()
# update membership for researchers, pis, owners, operators
self.update_driver_relations (record, new_record)
return 1
def fill_record_sfa_info(self, records):
def startswith(prefix, values):
return [value for value in values if value.startswith(prefix)]
# get person ids
person_ids = []
site_ids = []
for record in records:
person_ids.extend(record.get("person_ids", []))
site_ids.extend(record.get("site_ids", []))
if 'site_id' in record:
site_ids.append(record['site_id'])
# get all pis from the sites we've encountered
# and store them in a dictionary keyed on site_id
site_pis = {}
if site_ids:
pi_filter = {'|roles': ['pi'], '|site_ids': site_ids}
pi_list = self.shell.GetPersons(pi_filter, ['person_id', 'site_ids'])
for pi in pi_list:
# we will need the pi's hrns also
person_ids.append(pi['person_id'])
# we also need to keep track of the sites these pis
# belong to
for site_id in pi['site_ids']:
if site_id in site_pis:
site_pis[site_id].append(pi)
else:
site_pis[site_id] = [pi]
# get sfa records for all records associated with these records.
# we'll replace pl ids (person_ids) with hrns from the sfa records
# we obtain
# get the registry records
person_list, persons = [], {}
person_list = dbsession.query (RegRecord).filter(RegRecord.pointer.in_(person_ids))
# create a hrns keyed on the sfa record's pointer.
# Its possible for multiple records to have the same pointer so
# the dict's value will be a list of hrns.
persons = defaultdict(list)
for person in person_list:
persons[person.pointer].append(person)
# get the pl records
pl_person_list, pl_persons = [], {}
pl_person_list = self.shell.GetPersons(person_ids, ['person_id', 'roles'])
pl_persons = list_to_dict(pl_person_list, 'person_id')
# fill sfa info
for record in records:
# skip records with no pl info (top level authorities)
#if record['pointer'] == -1:
# continue
sfa_info = {}
type = record['type']
logger.info("fill_record_sfa_info - incoming record typed %s"%type)
if (type == "slice"):
# all slice users are researchers
record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
record['PI'] = []
record['researcher'] = []
for person_id in record.get('person_ids', []):
hrns = [person.hrn for person in persons[person_id]]
record['researcher'].extend(hrns)
# pis at the slice's site
if 'site_id' in record and record['site_id'] in site_pis:
pl_pis = site_pis[record['site_id']]
pi_ids = [pi['person_id'] for pi in pl_pis]
for person_id in pi_ids:
hrns = [person.hrn for person in persons[person_id]]
record['PI'].extend(hrns)
record['geni_creator'] = record['PI']
elif (type.startswith("authority")):
record['url'] = None
logger.info("fill_record_sfa_info - authority xherex")
if record['pointer'] != -1:
record['PI'] = []
record['operator'] = []
record['owner'] = []
for pointer in record.get('person_ids', []):
if pointer not in persons or pointer not in pl_persons:
# this means there is not sfa or pl record for this user
continue
hrns = [person.hrn for person in persons[pointer]]
roles = pl_persons[pointer]['roles']
if 'pi' in roles:
record['PI'].extend(hrns)
if 'tech' in roles:
record['operator'].extend(hrns)
if 'admin' in roles:
record['owner'].extend(hrns)
# xxx TODO: OrganizationName
elif (type == "node"):
sfa_info['dns'] = record.get("hostname", "")
# xxx TODO: URI, LatLong, IP, DNS
elif (type == "user"):
logger.info('setting user.email')
sfa_info['email'] = record.get("email", "")
sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
sfa_info['geni_certificate'] = record['gid']
# xxx TODO: PostalAddress, Phone
record.update(sfa_info)
def Resolve(self, api, xrns, type=None, details=False):
if not isinstance(xrns, types.ListType):
# try to infer type if not set and we get a single input
if not type:
type = Xrn(xrns).get_type()
xrns = [xrns]
hrns = [urn_to_hrn(xrn)[0] for xrn in xrns]
# load all known registry names into a prefix tree and attempt to find
# the longest matching prefix
# create a dict where key is a registry hrn and its value is a list
# of hrns at that registry (determined by the known prefix tree).
xrn_dict = {}
registries = api.registries
tree = prefixTree()
registry_hrns = registries.keys()
tree.load(registry_hrns)
for xrn in xrns:
registry_hrn = tree.best_match(urn_to_hrn(xrn)[0])
if registry_hrn not in xrn_dict:
xrn_dict[registry_hrn] = []
xrn_dict[registry_hrn].append(xrn)
records = []
for registry_hrn in xrn_dict:
# skip the hrn without a registry hrn
# XX should we let the user know the authority is unknown?
if not registry_hrn:
continue
# if the best match (longest matching hrn) is not the local registry,
# forward the request
xrns = xrn_dict[registry_hrn]
if registry_hrn != api.hrn:
credential = api.getCredential()
interface = api.registries[registry_hrn]
server_proxy = api.server_proxy(interface, credential)
# should propagate the details flag but that's not supported in the xmlrpc interface yet
#peer_records = server_proxy.Resolve(xrns, credential,type, details=details)
peer_records = server_proxy.Resolve(xrns, credential)
# pass foreign records as-is
# previous code used to read
# records.extend([SfaRecord(dict=record).as_dict() for record in peer_records])
# not sure why the records coming through xmlrpc had to be processed at all
records.extend(peer_records)
# try resolving the remaining unfound records at the local registry
local_hrns = list ( set(hrns).difference([record['hrn'] for record in records]) )
#
local_records = dbsession.query(RegRecord).filter(RegRecord.hrn.in_(local_hrns))
if type:
local_records = local_records.filter_by(type=type)
local_records=local_records.all()
for local_record in local_records:
augment_with_sfa_builtins (local_record)
logger.info("Resolve, (details=%s,type=%s) local_records=%s "%(details,type,local_records))
local_dicts = [ record.__dict__ for record in local_records ]
if details:
# in details mode we get as much info as we can, which involves contacting the
# testbed for getting implementation details about the record
self.driver.augment_records_with_testbed_info(local_dicts)
# also we fill the 'url' field for known authorities
# used to be in the driver code, sounds like a poorman thing though
def solve_neighbour_url (record):
if not record.type.startswith('authority'): return
hrn=record.hrn
for neighbour_dict in [ api.aggregates, api.registries ]:
if hrn in neighbour_dict:
record.url=neighbour_dict[hrn].get_url()
return
for record in local_records: solve_neighbour_url (record)
# convert local record objects to dicts for xmlrpc
# xxx somehow here calling dict(record) issues a weird error
# however record.todict() seems to work fine
# records.extend( [ dict(record) for record in local_records ] )
records.extend( [ record.todict(exclude_types=[InstrumentedList]) for record in local_records ] )
if not records:
raise RecordNotFound(str(hrns))
return records
