def delete(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).delete(setup)
errors = result['errors']
# Signal only Registry Errors
if errors:
raising = False
for err in errors:
if err['type'] == "Reg":
if "Record not found" in err['exception']:
raising = False
else:
raising = True
if raising:
raise SliceException(errors)
db.delete(dbconnection, 'slices', self.id)
for u in current['users']:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice delete()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.delete(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
# Remove slice from user
del u['slices'][self.id]
db.users(dbconnection, user, u)
# Update the Project of the slice
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = list(set(project['slices']) - set([self.id]))
db.projects(dbconnection, project)
# Warning if errors on AMs
#if errors:
# raise SliceWarningException(errors)
return True
def delete(self, dbconnection, setup=None):
# Get Project from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='projects', id=self.id)
result = super(Project, self).delete(setup)
errors = result['errors']
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise ProjectException(errors)
db.delete(dbconnection, 'projects', self.id)
for u in current['pi_users']:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
logger.debug("Update user %s after Project delete()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
# Slices will be removed by Sync
return True
def save(self, dbconnection, setup=None):
# Get Project from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='projects', id=self.id)
result = super(Project, self).save(setup)
errors = result['errors']
if errors:
raise ProjectException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Project created
if current is None:
db.projects(dbconnection, result)
current = db.get(dbconnection, table='projects', id=self.id)
# Update existing project
else:
db.projects(dbconnection, result, self.id)
# update pi_users after Save
pi_users = list(
set(current['pi_users']) | set(self.getAttribute('pi_users')))
for u in pi_users:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
logger.debug("Update user %s after Project save()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
# update slices after Save
slices = list(
set(current['slices']) | set(self.getAttribute('slices')))
if setup:
setup.setEndpoints(myslicelibsetup.endpoints)
for s in current['slices']:
sl = q(Slice, setup).id(s).get().first()
db.slices(dbconnection, sl.dict())
return True
def remove_resources(data, slice):
data_resources = [d['id'] if isinstance(d,dict) else d for d in data['resources']]
# check if we need to remove resources from the slice
for resource_id in [x['id'] for x in slice.resources]:
if resource_id not in data_resources:
# remove resource from slice
res = Resource(db.get(dbconnection, table='resources', id=resource_id))
slice.removeResource(res)
return slice
def remove_users(data, slice):
data_users = [d['id'] if isinstance(d,dict) else d for d in data['users']]
# check if we need to remove users from the slice
for user_id in slice.users:
if user_id not in data_users:
# remove user from slice
u = User(db.get(dbconnection, table='users', id=user_id))
slice.removeUser(u)
return slice
def add_resources(data, slice):
# check if the resources in the request are in the slice
for data_resource in data['resources']:
if isinstance(data_resource, dict):
res = Resource(data_resource)
else:
res = Resource(db.get(dbconnection, table='resources', id=data_resource))
if res.id not in [x['id'] for x in slice.resources]:
slice.addResource(res)
return slice
def add_users(data, slice):
# check if the users in the request are in the slice
for data_user in data['users']:
if isinstance(data_user, dict):
u = User(data_user)
else:
u = User(db.get(dbconnection, table='users', id=data_user))
if u.id not in slice.users:
slice.addUser(u)
return slice
def merge(self, dbconnection):
db_user = db.get(dbconnection, table='users', id=self.id)
if db_user:
if 'private_key' in db_user:
self.setAttribute('private_key', db_user['private_key'])
if 'public_key' in db_user:
self.setAttribute('public_key', db_user['public_key'])
if 'password' in db_user:
self.setAttribute('password', db_user['password'])
if 'generate_keys' in db_user:
self.setAttribute('generate_keys', db_user['generate_keys'])
return self
def save(self, dbconnection, setup=None):
logger.warning("User.save() called")
# Get User from local DB
current = db.get(dbconnection, table='users', id=self.id)
if self.getAttribute('generate_keys'):
private_key, public_key = generate_RSA()
self.setAttribute('private_key', private_key.decode('utf-8'))
self.setAttribute('public_key', public_key.decode('utf-8'))
self.appendAttribute('keys', self.getAttribute('public_key'))
# Once it is generated, turn this option off
self.setAttribute('generate_keys', False)
result = super(User, self).save(setup)
errors = result['errors']
# Raise exception to send the errors as logs
if errors:
raise UserException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# Update user in local DB
if not self.getAttribute('id'):
raise UserException(
"This user has no id, could not be saved in local DB: {}".
format(self))
logger.debug("DB user save()")
logger.debug(result)
if not self.getAttribute('id'):
logger.critical("-------> USER HAS NO ID <-------")
raise Exception(
"Trying to save a user that has no id will remove all data in users table!!!"
)
# New User created
if current is None:
db.users(dbconnection, result)
else:
db.users(dbconnection, result, self.getAttribute('id'))
return True
def delete(self, dbconnection, setup=None):
logger.debug("Delete Authority %s" % self.id)
# Get Authority from local DB
# to update the pi_users after Save
logger.debug("Get current object")
current = db.get(dbconnection, table='authorities', id=self.id)
logger.debug("Delete sent to myslicelib")
result = super(Authority, self).delete(setup)
logger.debug("result from myslicelib")
logger.debug(result)
errors = result['errors']
logger.debug("checking errors")
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise AuthorityException(errors)
logger.debug("Delete Authority from local DB")
db.delete(dbconnection, 'authorities', self.id)
logger.debug("Delete users of the Authority from local DB")
for u in current['users']:
logger.debug("Delete user %s" % u)
db.delete(dbconnection, 'users', u)
logger.debug("Update PI users of the Authority in local DB")
for u in current['pi_users']:
logger.debug("Get user %s" % u)
user = q(User).id(u).get().first()
if user:
logger.debug("Update user %s in Authority delete()" % u)
logger.debug(user)
user = user.merge(dbconnection)
db.users(dbconnection, user.dict(), user.id)
return True
def run(q):
"""
Manages newly created events
"""
logger.info("Worker activity events starting")
# db connection is shared between threads
dbconnection = connect()
while True:
try:
event = Event(q.get())
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with event: {}".format(e))
event.logError("Error in worker activity: {}".format(e))
event.setError()
dispatch(dbconnection, event)
continue
else:
logger.debug("%s - Manage event".format(event.id))
# TODO: if event.creatingObject()
# Check if the event.object.id already exists or not
# if it exists -> add a numer to the id & hrn to make it unique
if event.object.type == ObjectType.PASSWORD:
try:
event.setPending()
event.logInfo("Event is pending, please check your email")
logger.debug("Event %s is pending" % event.id)
except Exception as e:
logger.error(
"Error in processing Event (1): {}".format(event))
logger.error("Error while processing: {}".format(e))
event.logError(str(e))
continue
# Register a new object for a new user
# id should be generated into the web to avoid duplicates
elif event.isNew() and event.object.type in [
ObjectType.USER, ObjectType.AUTHORITY
] and event.user is None and event.creatingObject():
try:
# The user must confirm his/her email
logger.debug("Event Type: {}".format(type(event)))
event.setConfirm()
logger.debug("Event %s is confirm" % event.id)
event.logInfo(
"Event is expecting your confirmation, please check your email"
)
except Exception as e:
logger.error(
"Error in processing Event (2): {}".format(event))
logger.error("Error while processing: {}".format(e))
event.logError(str(e))
continue
else:
try:
logger.debug("%s - get user %s" % (event.id, event.user))
db_user = db.get(dbconnection,
table='users',
id=event.user)
if db_user:
user = User(db_user)
logger.debug("%s - Does user %s has privilege?" %
(event.id, event.user))
if user.has_privilege(event):
logger.debug("%s - setWaiting" % event.id)
event.logInfo("Event waiting to be processed")
event.setWaiting()
else:
logger.debug("%s - setPending" % event.id)
event.logInfo(
"your user has no rights on %s, event pending until approved"
% event.user)
event.setPending()
else:
event.setError()
logger.error("User {} not found in event {}".format(
event.user, event.id))
event.logError("User %s not found" % event.user)
# raising an Exception here, blocks the REST API
#raise Exception("User %s not found" % event.user)
except Exception as e:
logger.error("Error processing Event")
logger.error(event)
import traceback
logger.error(traceback.print_exc())
traceback.print_exc()
event.setError()
event.logError(str(e))
logger.error("Unable to fetch the user {} from db".format(
event.user))
logger.exception(e)
continue
# dispatch the updated event
dispatch(dbconnection, event)
def events_run(lock, qUserEvents):
"""
Process the user after approval
"""
logger.info("Worker users events starting")
dbconnection = connect()
while True:
try:
event = Event(qUserEvents.get())
except Exception as e:
logger.error("Problem with event: {}".format(e))
event.logError(str(e))
event.setError()
db.dispatch(dbconnection, event)
continue
else:
logger.info("Processing event from user {}".format(event.user))
with lock:
try:
event.setRunning()
event.logInfo("Event is running")
logger.debug("Event %s is running" % event.id)
isSuccess = False
# If we generate a new key pair the Query will not work, use the myslice account for that
if event.user and hasattr(
event.object, 'generate_keys'
) and event.object.generate_keys is False:
u = User(
db.get(dbconnection, table='users', id=event.user))
user_setup = UserSetup(u, myslicelibsetup.endpoints)
else:
user_setup = None
##
# Creating a new user
if event.creatingObject():
logger.info("Creating user {}".format(event.object.id))
user = User(event.data)
isSuccess = user.save(dbconnection, user_setup)
##
# Deleting user
if event.deletingObject():
logger.info("Deleting user {}".format(event.object.id))
user = User(
db.get(dbconnection,
table='users',
id=event.object.id))
if not user:
raise Exception("User doesn't exist")
user.id = event.object.id
isSuccess = user.delete(dbconnection, user_setup)
##
# Updating user
if event.updatingObject():
logger.info("Updating user {}".format(event.object.id))
user = User(event.data)
local_user = User(
db.get(dbconnection,
table='users',
id=event.object.id))
# we don't allow users to change their email
user.email = local_user.email
user.id = event.object.id
isSuccess = user.save(dbconnection, user_setup)
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem updating user: {} - {}".format(
event.object.id, e))
event.logError(str(e))
event.setError()
continue
if isSuccess:
event.setSuccess()
event.logInfo("Event success")
logger.debug("Event %s Success" % event.id)
else:
logger.error("Error event {}: action failed".format(
event.id))
event.setError()
event.logError("Error in worker users: action failed")
##
# we then dispatch the event
db.dispatch(dbconnection, event)
def syncUsers(lock, email=None):
# DB connection
dbconnection = db.connect()
# acquires lock
with lock:
logger.info("Worker users starting synchronization")
try:
if email:
users = q(User).filter('email', email).get()
else:
users = q(User).get()
"""
update local user table
"""
if len(users) > 0:
# Add users from Registry unkown from local DB
# this is used to bootstrap with init_user script
for u in users:
logger.debug("looking for {} in local db".format(u.id))
if not db.get(dbconnection, table='users', id=u.id):
local_users = db.get(dbconnection, table='users')
logger.warning("Number of users in local db: %s" %
len(local_users))
#print("this user is not in local db, add it")
logger.info("Found new user from Registry: %s" % u.id)
#logger.info("We don't add the missing users yet, as portal is the single point of entry")
db.users(dbconnection, u.dict())
local_users = db.get(dbconnection, table='users')
# Update users known in local DB
for lu in local_users:
logger.info("Synchronize user %s" % lu['id'])
try:
# add status if not present and update on db
if not 'status' in lu:
lu['status'] = Status.ENABLED
lu['enabled'] = format_date()
if not users.has(lu['id']) and lu[
'status'] is not Status.PENDING:
# delete user that has been deleted in Reg
db.delete(dbconnection, 'users', lu['id'])
logger.info("User {} deleted".format(lu['id']))
else:
remote_user = next(
(item
for item in users if item.id == lu['id']),
False)
if remote_user:
# merge fields of local user with remote
# keep local values for
# password, private_key, public_key and generate_keys
updated_user = remote_user.merge(dbconnection)
updated_user = updated_user.dict()
# if user has private key
# update its Credentials
if 'private_key' in updated_user and updated_user[
'private_key'] is not None:
updated_user = update_credentials(
dbconnection, updated_user)
# Update user
#logger.debug("Update user %s" % updated_user['id'])
db.users(dbconnection, updated_user,
updated_user['id'])
except Exception as e:
logger.warning("Could not synchronize user %s" %
lu['id'])
logger.exception(e)
raise
else:
logger.warning(
"Query users is empty, check myslicelib and the connection with SFA Registry"
)
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(e)
raise
logger.info("Worker users finished period synchronization")
def emails_run(qEmails):
"""
Process Requests and send Emails accordingly
"""
# db connection is shared between threads
dbconnection = connect()
while True:
try:
event = Event(qEmails.get())
except Exception as e:
logger.error("Problem with event: {}".format(e))
continue
else:
if event.notify:
# We try to send the email only once
event.notify = False
dispatch(dbconnection, event)
# Recipients
# TODO: Send specific emails
# Status did NOT changed
# Comments about an event with a message
if event.status == event.previous_status:
logger.warning("TODO: send specific emails with messages")
recipients = set()
url = s.web['url']
if s.web['port'] and s.web['port'] != 80:
url = url +':'+ s.web['port']
buttonLabel = "View details"
if event.object.type == ObjectType.PASSWORD:
recipients.add(User(event.object.id))
url = url+'/password/'+event.data['hashing']
subject, template = build_subject_and_template('password', event)
buttonLabel = "Change password"
else:
if event.isPending():
# Find the authority of the event object
# Then according the authority, put the pi_emails in pis_email
try:
authority_id = event.data['authority']
except KeyError:
msg = 'Authority id not specified ({})'.format(event.id)
logger.error(msg)
event.logWarning('Authority not specified in event {}, email not sent'.format(event.id))
pass
else:
authority = Authority(db.get(dbconnection, table='authorities', id=authority_id))
if not authority:
# get admin users
users = db.get(dbconnection, table='users')
for u in users:
user = User(u)
if user.isAdmin():
logger.debug("user %s is admin" % user.id)
recipients.add(user)
else:
for pi_id in authority.pi_users:
recipients.add(User(pi_id))
if not recipients:
msg = 'Emails cannot be sent because no one is the PI of {}'.format(event.object.id)
logger.error(msg)
event.logWarning('No recipients could be found for event {}, email not sent'.format(event.id))
subject, template = build_subject_and_template('request', event)
buttonLabel = "Approve / Deny"
url = url + '/activity'
else:
if event.user:
recipients.add(User(event.user))
else:
# Look for the user email in the Event
if event.object.type == ObjectType.USER:
recipients.add(User({'email':event.data['email'], 'first_name':event.data['first_name'], 'last_name':event.data['last_name']}))
elif event.object.type == ObjectType.AUTHORITY:
for user in event.data['users']:
recipients.add(User(user))
else:
for user in event.data['pi_users']:
recipients.add(User(user))
if event.isSuccess():
subject, template = build_subject_and_template('approve', event)
elif event.isDenied():
subject, template = build_subject_and_template('deny', event)
try:
sendEmail(event, recipients, subject, template, url, buttonLabel)
except Exception as e:
import traceback
traceback.print_exc()
msg = "Error in event {} while trying to send an email: {} {}".format(event.id, e, traceback.print_exc())
logger.error(msg)
event.logWarning(msg)
continue
finally:
dispatch(dbconnection, event)
def events_run(lock, qProjectEvents):
"""
Process the authority after approval
"""
logger.info("Worker authorities events starting")
# db connection is shared between threads
dbconnection = connect()
while True:
try:
event = Event(qProjectEvents.get())
except Exception as e:
logger.error("Problem with event: {}".format(e))
event.logError(str(e))
event.setError()
db.dispatch(dbconnection, event)
continue
else:
logger.info("Processing event: {} from user {}".format(event.id, event.user))
with lock:
try:
if event.isApproved():
user_id = event.manager
else:
user_id = event.user
u = User(db.get(dbconnection, table='users', id=user_id))
# Registry Only
user_setup = UserSetup(u, myslicelibsetup.registry_endpoints)
event.setRunning()
event.logInfo("Event is running")
logger.debug("Event %s is running" % event.id)
isSuccess = False
if event.creatingObject() or event.updatingObject():
logger.info("creating or updating the object project {}".format(event.data))
proj = Project(event.data)
proj.id = event.object.id
# XXX CASES TO BE CHECKED
if event.user not in proj.pi_users:
pi = User(db.get(dbconnection, table='users', id=event.user))
proj.addPi(pi)
isSuccess = proj.save(dbconnection, user_setup)
else:
p = db.get(dbconnection, table='projects', id=event.object.id)
if not p:
raise Exception("Project doesn't exist")
proj = Project(p)
if event.deletingObject():
logger.info("deleting the object project {}".format(event.object.id))
isSuccess = proj.delete(dbconnection, user_setup)
if event.addingObject():
logger.info("adding data to the object project {}".format(event.object.id))
if event.data.type == DataType.USER:
logger.info("Project only supports PI at the moment, need new feature in SFA Reg")
if event.data.type == DataType.PI or event.data.type == DataType.USER:
for val in event.data.values:
pi = User(db.get(dbconnection, table='users', id=val))
proj.addPi(pi)
isSuccess = proj.save(dbconnection, user_setup)
if event.removingObject():
logger.info("removing data from the object project {}".format(event.object.id))
if event.data.type == DataType.USER:
logger.info("Project only supports PI at the moment, need new feature in SFA Reg")
if event.data.type == DataType.PI or event.data.type == DataType.USER:
for val in event.data.values:
pi = User(db.get(dbconnection, table='users', id=val))
proj.removePi(pi)
isSuccess = proj.save(dbconnection, user_setup)
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with event {}: {}".format(event.id,e))
event.logError("Error in worker projects: {}, traceback: {}".format(e,traceback.print_exc()))
event.setError()
continue
else:
if isSuccess:
event.setSuccess()
event.logInfo("Event success")
logger.debug("Event %s Success" % event.id)
else:
logger.error("Error event {}: action failed".format(event.id))
event.setError()
event.logError("Error in worker projects: action failed")
finally:
db.dispatch(dbconnection, event)
def events_run(qSliceEvents):
"""
Process the slice after approval
"""
logger.info("Worker slices events starting")
while True:
try:
event = Event(qSliceEvents.get())
except Exception as e:
logger.error("Problem with event: {}".format(e))
event.logError(str(e))
event.setError()
db.dispatch(dbconnection, event)
continue
else:
logger.info("Processing event {} from user {}".format(event.id, event.user))
with lock:
try:
event.setRunning()
isSuccess = False
u = User(db.get(dbconnection, table='users', id=event.user))
user_setup = UserSetup(u, myslicelibsetup.endpoints)
if event.creatingObject():
sli = Slice(event.data)
# XXX To be checked
sli.id = event.object.id
# Add all Project PIs to the Slice
project = db.get(dbconnection, table='projects', id=sli.project)
for us in project['pi_users']:
us = User(db.get(dbconnection, table='users', id=us))
sli.addUser(us)
if 'users' in event.data and 'geni_users' not in event.data:
for u_id in event.data['users']:
u = User(db.get(dbconnection, table='users', id=u_id))
sli.addUser(u)
# Take into account the Resources on Create
if 'resources' in event.data:
for resource in event.data['resources']:
if not isinstance(resource, dict):
res = db.get(dbconnection, table='resources', id=resource)
if not res:
raise Exception("Resource %s doesn't exist" % resource)
resource = res
sli.addResource(Resource(resource))
# expiration_date = Renew at AMs
isSuccess = sli.save(dbconnection, user_setup)
if event.updatingObject():
logger.debug("Update users / resources of Slice %s" % event.object.id)
sli = Slice(db.get(dbconnection, table='slices', id=event.object.id))
## adding users
sli = add_users(event.data, sli)
## removing users
sli = remove_users(event.data, sli)
## adding resources
sli = add_resources(event.data, sli)
## removing resources
sli = remove_resources(event.data, sli)
isSuccess = sli.save(dbconnection, user_setup)
if event.deletingObject():
sli = Slice(db.get(dbconnection, table='slices', id=event.object.id))
if not sli:
raise Exception("Slice doesn't exist")
isSuccess = sli.delete(dbconnection, user_setup)
if event.addingObject():
sli = Slice(db.get(dbconnection, table='slices', id=event.object.id))
if event.data.type == DataType.USER:
for val in event.data.values:
if isinstance(val, dict):
val = val['id']
u = User(db.get(dbconnection, table='users', id=val))
sli.addUser(u)
isSuccess = sli.save(dbconnection, user_setup)
if event.data['type'] == DataType.RESOURCE:
for val in event.data.values:
if isinstance(val, dict):
val = val['id']
res = Resource(db.get(dbconnection, table='resources', id=val))
sli.addResource(res)
isSuccess = sli.save(dbconnection, user_setup)
if event.removingObject():
sli = Slice(db.get(dbconnection, table='slices', id=event.object.id))
if event.data.type == DataType.USER:
for val in event.data['values']:
if isinstance(val, dict):
val = val['id']
u = User(db.get(dbconnection, table='users', id=val))
sli.removeUser(u)
isSuccess = sli.save(dbconnection, user_setup)
if event.data.type == DataType.RESOURCE:
for val in event.data.values:
if isinstance(val, dict):
val = val['id']
r = Resource(db.get(dbconnection, table='resources', id=val))
sli.removeResource(r)
isSuccess = sli.save(dbconnection, user_setup)
except SliceException as e:
# CREATE, UPDATE, DELETE
# Calls toward Registry
# If an AM sends an Error it is not blocking
if event.creatingObject() or event.updatingObject() or event.deletingObject():
logger.debug("DEBUG services worker slices")
for err in e.stack:
logger.debug(err)
if err['type'] == 'Reg':
event.setError()
break
else:
# XXX TO BE REFINED
event.setSuccess()
#event.setWarning()
# TODO:
# if ALL AMs have failed -> Error
# if One AM succeeded -> Warning
else:
# XXX TO BE REFINED
logger.debug("DEBUG services worker slices")
for err in e.stack:
logger.debug(err)
event.setWarning()
#event.setError()
continue
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with event: {}".format(e))
event.logError(str(e))
event.setError()
continue
else:
if isSuccess:
event.setSuccess()
else:
event.setError()
db.dispatch(dbconnection, event)
def syncSlices(id=None):
with lock:
logger.info("Worker slices starting synchronization")
try:
# DB connection
dbconnection = db.connect()
# Update an existing Slice
if id:
slices = Slices([Slice(db.get(dbconnection, table='slices', id=id))])
# MySliceLib Query Slices
else:
slices = q(Slice).get()
if len(slices)==0:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
# ------------------------------------------------------
# Synchronize resources of a Slice at AMs
# !!! only if the slice_id is specified !!!
# Otherwise it is way too long to synchronize all slices
# ------------------------------------------------------
# TODO: trigger this function in background for a user
# that want to refresh his slice / when he selected one
# ------------------------------------------------------
if id:
for slice in slices:
if len(slice.users) > 0:
try:
u = User(db.get(dbconnection, table='users', id=slice.users[0]))
logger.info("Synchronize slice %s:" % slice.hrn)
# Synchronize resources of the slice only if we have the user's private key or its credentials
# XXX Should use delegated Credentials
#if (hasattr(u,'private_key') and u.private_key is not None and len(u.private_key)>0) or (hasattr(u,'credentials') and len(u.credentials)>0):
if u.private_key or (hasattr(u,'credentials') and len(u.credentials)>0):
user_setup = UserSetup(u,myslicelibsetup.endpoints)
logger.info("Slice.id(%s).get() with user creds" % slice.hrn)
s = q(Slice, user_setup).id(slice.id).get().first()
db.slices(dbconnection, s.dict(), slice.id)
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with slice %s" % slice.id)
logger.exception(str(e))
raise
else:
logger.info("slice %s has no users" % slice.hrn)
# update local slice table
else:
if len(slices)>0:
local_slices = db.slices(dbconnection)
# Add slices from Registry unkown from local DB
for s in slices:
if not db.get(dbconnection, table='slices', id=s.id):
logger.info("Found new slice from Registry: %s" % s.id)
db.slices(dbconnection, s.dict(), s.id)
# Update slices known in local DB
for ls in local_slices :
logger.info("Synchronize Slice {}".format(ls['id']))
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
if not slices.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete slices that have been deleted elsewhere
db.delete(dbconnection, 'slices', ls['id'])
logger.info("Slice {} deleted".format(ls['id']))
else:
db.slices(dbconnection, ls, ls['id'])
else:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(str(e))
raise
logger.info("Worker slices finished period synchronization")
def save(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).save(setup)
errors = result['errors']
result = {**(self.dict()), **result['data'][0]}
if not errors:
for r in result['resources']:
if (not 'services' in r) or (not r['services']):
logger.warning("result from slice.save didn't had login info")
logger.warning("sleeping 10s before asking again to AMs")
import time
time.sleep(10)
slice = q(Slice, setup).id(self.id).get().first()
result = slice.dict()
break
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Slice created
if current is None:
db.slices(dbconnection, result)
current = db.get(dbconnection, table='slices', id=self.id)
# Update existing slice
else:
db.slices(dbconnection, result, self.id)
# Update users both previously and currently in Slice
users = list(set(current['users']) | set(self.getAttribute('users')))
for u in users:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice save()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.save(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
if u in current['users'] and u not in self.getAttribute('users'):
# Remove slice from user
del u['slices'][self.id]
elif u not in current['users'] and u in self.getAttribute('users'):
# Add slice to user
u['slice'].append(self.id)
db.users(dbconnection, user, u)
# Update the Project of the slice
logger.debug("cooko slice: {}".format(self))
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = project['slices'] + [self.id]
db.projects(dbconnection, project)
# Insert / Delete Leases if necessary
if self.hasLeases:
flag = -1
for lease in self.leases:
# No resources reserved
if len(result['leases'])==0:
flag = -1
# All resources of a Lease have been succesfully reserved
elif lease['resources'] == result['leases'][0]['resources']:
flag = 0
# Some Resources of a Lease have been reserved
elif len(set(lease['resources']).intersection(set(result['leases'][0]['resources']))) > 0:
db.leases(dbconnection, lease)
flag = 1
for lease in self.removedLeases:
if lease not in result['leases']:
db.delete(dbconnection, 'leases', lease.id)
flag = False
if flag == -1:
errors.append("No reservation has been accepted by the testbeds")
elif flag == 1:
errors.append("Some resources have been reserved others were unavailable")
raise SliceWarningException(errors)
if errors:
raise SliceException(errors)
else:
return True
def events_run(lock, qLeasesEvents):
"""
Process the leases events
"""
logger.info("Worker leases events starting")
# db connection is shared between threads
dbconnection = connect()
while True:
try:
event = Event(qLeasesEvents.get())
except Exception as e:
logger.error("Problem with event: {}".format(e))
event.logError("Error in worker leases: {}".format(e))
event.setError()
db.dispatch(dbconnection, event)
continue
else:
logger.info("Processing event from user {}".format(event.user))
try:
event.setRunning()
event.logInfo("Event is running")
logger.debug("Event %s is running" % event.id)
isSuccess = False
u = User(db.get(dbconnection, table='users', id=event.user))
user_setup = UserSetup(u, myslicelibsetup.endpoints)
if event.creatingObject():
leases = []
if isinstance(event.data, list):
for l in event.data:
slice_id = l['slice_id']
leases.append(Lease(l))
else:
slice_id = event.data['slice_id']
leases.append(Lease(event.data))
sli = Slice(
db.get(dbconnection, table='slices', id=slice_id))
if not sli:
raise Exception("Slice doesn't exist")
for lease in leases:
for val in lease.resources:
r = db.get(dbconnection, table='resources', id=val)
# Add resource only if it exists in DB
if r is not None:
r = Resource(r)
sli.addResource(r)
else:
r = Resource({'id': val})
lease.removeResource(r)
if len(lease.resources) > 0:
sli.addLease(lease)
else:
raise Exception("Invalid resources")
isSuccess = sli.save(dbconnection, user_setup)
if event.deletingObject():
lease = Lease(
db.get(dbconnection,
table='leases',
id=event.object.id))
if not lease:
raise Exception("Lease doesn't exist")
sli = Slice(
db.get(dbconnection,
table='slices',
id=event.data['slice_id']))
if not sli:
raise Exception("Slice doesn't exist")
for val in event.data['resources']:
r = Resource(
db.get(dbconnection, table='resources', id=val))
# Remove resource only if it exists in DB
if r:
sli.removeResource(r)
else:
r = Resource({'id': val})
lease.removeResource(r)
sli.removeLease(lease)
isSuccess = sli.save(dbconnection, user_setup)
except SliceException as e:
# CREATE, DELETE
# If at least one of the AMs replies with success, it's ok
# If all AMs have failed -> Error
for err in e.stack:
event.logError("Error in worker leases: {}".format(err))
logger.error("Error in worker leases: {}".format(err))
# XXX TO BE REFINED
event.setError()
continue
except SliceWarningException as e:
for err in e.stack:
event.logWarning(str(err))
logger.warning(str(err))
event.setWarning()
continue
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with event {}: {}".format(event.id, e))
event.logError("Error in worker leases: {}".format(e))
event.setError()
continue
else:
if isSuccess:
event.setSuccess()
event.logInfo("Event success")
logger.debug("Event %s Success" % event.id)
else:
logger.error("Error event {}: action failed".format(
event.id))
event.setError()
event.logError("Error in worker leases: action failed")
db.dispatch(dbconnection, event)
def save(self, dbconnection, setup=None):
# Get Authority from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='authorities', id=self.id)
new_users = self.handleDict('users')
new_pi_users = self.handleDict('pi_users')
result = super(Authority, self).save(setup)
errors = result['errors']
if errors:
raise AuthorityException(errors)
logger.debug(result)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Authority created
if current is None:
db.authorities(dbconnection, result)
current = db.get(dbconnection, table='authorities', id=self.id)
# Update existing authority
else:
db.authorities(dbconnection, result, self.id)
# Create new users under a New Authority
# Otherwise users are created with User.save()
for u in new_users:
if isinstance(u, dict):
if not "email" in u:
raise Warning("user has no email can not be created")
user = db.get(dbconnection,
table='users',
filter={'email': u['email']},
limit=1)
# If the user doesn't exit, create it
if not user:
user = User(u)
user.setAttribute('authority', self.id)
userCreated = user.save(dbconnection)
if not userCreated:
raise Warning("user has not been created")
self.users.append(user.id)
modified = True
# Grant or Revoke PI Rights
current['pi_users'] = current.get('pi_users', [])
pi_users = current['pi_users'] + self.getAttribute(
'pi_users') + new_pi_users
modified = False
for u in pi_users:
if isinstance(u, dict):
if not "email" in u:
raise Warning(
"pi_user has no email can not be added as PI")
user = db.get(dbconnection,
table='users',
filter={'email': u['email']},
limit=1)
if not user:
raise Warning("user does not exist, can't be PI")
else:
user = User(user[0])
else:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
# REMOVE PI Rights
if user.id not in self.getAttribute('pi_users') and not any(
d['email'] == user.email for d in new_pi_users):
self.removePi(user)
modified = True
# GRANT PI Rights
elif user.id not in current['pi_users']:
self.addPi(user)
modified = True
logger.debug("Update user %s in Authority save()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
if modified:
result = super(Authority, self).save(setup)
errors = result['errors']
if errors:
raise AuthorityException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
db.authorities(dbconnection, result, self.id)
return True
