def syncProjects(lock):
# DB connection
dbconnection = connect()
# acquires lock
with lock:
logger.info("Worker projects starting synchronization")
# MySliceLib Query Slices
p = q(Project).get()
# update local projects table
if len(p)>0:
lprojects = db.projects(dbconnection, p.dict())
for ls in lprojects :
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
db.projects(dbconnection, ls)
if not p.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete projects that have been deleted elsewhere
db.delete(dbconnection, 'projects', ls['id'])
logger.info("Project {} deleted".format(ls['id']))
else:
logger.warning("Query projects is empty, check myslicelib and the connection with SFA Registry")
def delete(self, dbconnection, setup=None):
# Get Project from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='projects', id=self.id)
result = super(Project, self).delete(setup)
errors = result['errors']
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise ProjectException(errors)
db.delete(dbconnection, 'projects', self.id)
for u in current['pi_users']:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
logger.debug("Update user %s after Project delete()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
# Slices will be removed by Sync
return True
def delete(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).delete(setup)
errors = result['errors']
# Signal only Registry Errors
if errors:
raising = False
for err in errors:
if err['type'] == "Reg":
if "Record not found" in err['exception']:
raising = False
else:
raising = True
if raising:
raise SliceException(errors)
db.delete(dbconnection, 'slices', self.id)
for u in current['users']:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice delete()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.delete(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
# Remove slice from user
del u['slices'][self.id]
db.users(dbconnection, user, u)
# Update the Project of the slice
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = list(set(project['slices']) - set([self.id]))
db.projects(dbconnection, project)
# Warning if errors on AMs
#if errors:
# raise SliceWarningException(errors)
return True
def delete(self, dbconnection, setup=None):
logger.warning("User.delete() called")
logger.debug(self.getAttribute('id'))
result = super(User, self).delete(setup)
errors = result['errors']
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise UserException(errors)
db.delete(dbconnection, 'users', self.getAttribute('id'))
return True
def delete(self, dbconnection, setup=None):
logger.debug("Delete Authority %s" % self.id)
# Get Authority from local DB
# to update the pi_users after Save
logger.debug("Get current object")
current = db.get(dbconnection, table='authorities', id=self.id)
logger.debug("Delete sent to myslicelib")
result = super(Authority, self).delete(setup)
logger.debug("result from myslicelib")
logger.debug(result)
errors = result['errors']
logger.debug("checking errors")
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise AuthorityException(errors)
logger.debug("Delete Authority from local DB")
db.delete(dbconnection, 'authorities', self.id)
logger.debug("Delete users of the Authority from local DB")
for u in current['users']:
logger.debug("Delete user %s" % u)
db.delete(dbconnection, 'users', u)
logger.debug("Update PI users of the Authority in local DB")
for u in current['pi_users']:
logger.debug("Get user %s" % u)
user = q(User).id(u).get().first()
if user:
logger.debug("Update user %s in Authority delete()" % u)
logger.debug(user)
user = user.merge(dbconnection)
db.users(dbconnection, user.dict(), user.id)
return True
def syncSlices(id=None):
with lock:
logger.info("Worker slices starting synchronization")
try:
# DB connection
dbconnection = db.connect()
# Update an existing Slice
if id:
slices = Slices([Slice(db.get(dbconnection, table='slices', id=id))])
# MySliceLib Query Slices
else:
slices = q(Slice).get()
if len(slices)==0:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
# ------------------------------------------------------
# Synchronize resources of a Slice at AMs
# !!! only if the slice_id is specified !!!
# Otherwise it is way too long to synchronize all slices
# ------------------------------------------------------
# TODO: trigger this function in background for a user
# that want to refresh his slice / when he selected one
# ------------------------------------------------------
if id:
for slice in slices:
if len(slice.users) > 0:
try:
u = User(db.get(dbconnection, table='users', id=slice.users[0]))
logger.info("Synchronize slice %s:" % slice.hrn)
# Synchronize resources of the slice only if we have the user's private key or its credentials
# XXX Should use delegated Credentials
#if (hasattr(u,'private_key') and u.private_key is not None and len(u.private_key)>0) or (hasattr(u,'credentials') and len(u.credentials)>0):
if u.private_key or (hasattr(u,'credentials') and len(u.credentials)>0):
user_setup = UserSetup(u,myslicelibsetup.endpoints)
logger.info("Slice.id(%s).get() with user creds" % slice.hrn)
s = q(Slice, user_setup).id(slice.id).get().first()
db.slices(dbconnection, s.dict(), slice.id)
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with slice %s" % slice.id)
logger.exception(str(e))
raise
else:
logger.info("slice %s has no users" % slice.hrn)
# update local slice table
else:
if len(slices)>0:
local_slices = db.slices(dbconnection)
# Add slices from Registry unkown from local DB
for s in slices:
if not db.get(dbconnection, table='slices', id=s.id):
logger.info("Found new slice from Registry: %s" % s.id)
db.slices(dbconnection, s.dict(), s.id)
# Update slices known in local DB
for ls in local_slices :
logger.info("Synchronize Slice {}".format(ls['id']))
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
if not slices.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete slices that have been deleted elsewhere
db.delete(dbconnection, 'slices', ls['id'])
logger.info("Slice {} deleted".format(ls['id']))
else:
db.slices(dbconnection, ls, ls['id'])
else:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(str(e))
raise
logger.info("Worker slices finished period synchronization")
def save(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).save(setup)
errors = result['errors']
result = {**(self.dict()), **result['data'][0]}
if not errors:
for r in result['resources']:
if (not 'services' in r) or (not r['services']):
logger.warning("result from slice.save didn't had login info")
logger.warning("sleeping 10s before asking again to AMs")
import time
time.sleep(10)
slice = q(Slice, setup).id(self.id).get().first()
result = slice.dict()
break
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Slice created
if current is None:
db.slices(dbconnection, result)
current = db.get(dbconnection, table='slices', id=self.id)
# Update existing slice
else:
db.slices(dbconnection, result, self.id)
# Update users both previously and currently in Slice
users = list(set(current['users']) | set(self.getAttribute('users')))
for u in users:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice save()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.save(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
if u in current['users'] and u not in self.getAttribute('users'):
# Remove slice from user
del u['slices'][self.id]
elif u not in current['users'] and u in self.getAttribute('users'):
# Add slice to user
u['slice'].append(self.id)
db.users(dbconnection, user, u)
# Update the Project of the slice
logger.debug("cooko slice: {}".format(self))
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = project['slices'] + [self.id]
db.projects(dbconnection, project)
# Insert / Delete Leases if necessary
if self.hasLeases:
flag = -1
for lease in self.leases:
# No resources reserved
if len(result['leases'])==0:
flag = -1
# All resources of a Lease have been succesfully reserved
elif lease['resources'] == result['leases'][0]['resources']:
flag = 0
# Some Resources of a Lease have been reserved
elif len(set(lease['resources']).intersection(set(result['leases'][0]['resources']))) > 0:
db.leases(dbconnection, lease)
flag = 1
for lease in self.removedLeases:
if lease not in result['leases']:
db.delete(dbconnection, 'leases', lease.id)
flag = False
if flag == -1:
errors.append("No reservation has been accepted by the testbeds")
elif flag == 1:
errors.append("Some resources have been reserved others were unavailable")
raise SliceWarningException(errors)
if errors:
raise SliceException(errors)
else:
return True
def syncUsers(lock, email=None):
# DB connection
dbconnection = db.connect()
# acquires lock
with lock:
logger.info("Worker users starting synchronization")
try:
if email:
users = q(User).filter('email', email).get()
else:
users = q(User).get()
"""
update local user table
"""
if len(users) > 0:
# Add users from Registry unkown from local DB
# this is used to bootstrap with init_user script
for u in users:
logger.debug("looking for {} in local db".format(u.id))
if not db.get(dbconnection, table='users', id=u.id):
local_users = db.get(dbconnection, table='users')
logger.warning("Number of users in local db: %s" %
len(local_users))
#print("this user is not in local db, add it")
logger.info("Found new user from Registry: %s" % u.id)
#logger.info("We don't add the missing users yet, as portal is the single point of entry")
db.users(dbconnection, u.dict())
local_users = db.get(dbconnection, table='users')
# Update users known in local DB
for lu in local_users:
logger.info("Synchronize user %s" % lu['id'])
try:
# add status if not present and update on db
if not 'status' in lu:
lu['status'] = Status.ENABLED
lu['enabled'] = format_date()
if not users.has(lu['id']) and lu[
'status'] is not Status.PENDING:
# delete user that has been deleted in Reg
db.delete(dbconnection, 'users', lu['id'])
logger.info("User {} deleted".format(lu['id']))
else:
remote_user = next(
(item
for item in users if item.id == lu['id']),
False)
if remote_user:
# merge fields of local user with remote
# keep local values for
# password, private_key, public_key and generate_keys
updated_user = remote_user.merge(dbconnection)
updated_user = updated_user.dict()
# if user has private key
# update its Credentials
if 'private_key' in updated_user and updated_user[
'private_key'] is not None:
updated_user = update_credentials(
dbconnection, updated_user)
# Update user
#logger.debug("Update user %s" % updated_user['id'])
db.users(dbconnection, updated_user,
updated_user['id'])
except Exception as e:
logger.warning("Could not synchronize user %s" %
lu['id'])
logger.exception(e)
raise
else:
logger.warning(
"Query users is empty, check myslicelib and the connection with SFA Registry"
)
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(e)
raise
logger.info("Worker users finished period synchronization")