def syncProjects(lock):
# DB connection
dbconnection = connect()
# acquires lock
with lock:
logger.info("Worker projects starting synchronization")
# MySliceLib Query Slices
p = q(Project).get()
# update local projects table
if len(p)>0:
lprojects = db.projects(dbconnection, p.dict())
for ls in lprojects :
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
db.projects(dbconnection, ls)
if not p.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete projects that have been deleted elsewhere
db.delete(dbconnection, 'projects', ls['id'])
logger.info("Project {} deleted".format(ls['id']))
else:
logger.warning("Query projects is empty, check myslicelib and the connection with SFA Registry")
def _log(self, type='info', message=None):
if message:
self['log'].append({
'type': type,
'timestamp': format_date(),
'message': message
})
else:
return [el for el in self['log'] if el['type'] == type]
def save(self, dbconnection, setup=None):
# Get Project from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='projects', id=self.id)
result = super(Project, self).save(setup)
errors = result['errors']
if errors:
raise ProjectException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Project created
if current is None:
db.projects(dbconnection, result)
current = db.get(dbconnection, table='projects', id=self.id)
# Update existing project
else:
db.projects(dbconnection, result, self.id)
# update pi_users after Save
pi_users = list(
set(current['pi_users']) | set(self.getAttribute('pi_users')))
for u in pi_users:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
logger.debug("Update user %s after Project save()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
# update slices after Save
slices = list(
set(current['slices']) | set(self.getAttribute('slices')))
if setup:
setup.setEndpoints(myslicelibsetup.endpoints)
for s in current['slices']:
sl = q(Slice, setup).id(s).get().first()
db.slices(dbconnection, sl.dict())
return True
def save(self, dbconnection, setup=None):
logger.warning("User.save() called")
# Get User from local DB
current = db.get(dbconnection, table='users', id=self.id)
if self.getAttribute('generate_keys'):
private_key, public_key = generate_RSA()
self.setAttribute('private_key', private_key.decode('utf-8'))
self.setAttribute('public_key', public_key.decode('utf-8'))
self.appendAttribute('keys', self.getAttribute('public_key'))
# Once it is generated, turn this option off
self.setAttribute('generate_keys', False)
result = super(User, self).save(setup)
errors = result['errors']
# Raise exception to send the errors as logs
if errors:
raise UserException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# Update user in local DB
if not self.getAttribute('id'):
raise UserException(
"This user has no id, could not be saved in local DB: {}".
format(self))
logger.debug("DB user save()")
logger.debug(result)
if not self.getAttribute('id'):
logger.critical("-------> USER HAS NO ID <-------")
raise Exception(
"Trying to save a user that has no id will remove all data in users table!!!"
)
# New User created
if current is None:
db.users(dbconnection, result)
else:
db.users(dbconnection, result, self.getAttribute('id'))
return True
def get(self, id=None, p=None):
"""
- GET /finterop/sessions/<id|hrn>/(start|stop)
:return:
"""
response = []
current_user = self.get_current_user()
logger.info('get session')
if not current_user:
self.userError('not authenticated ')
return
# Security: only allow to watch sessions for which the user has rights (one of his slices)
# --------------------------------------
# XXX DISABLED for development
# --------------------------------------
#if id not in current_user['slices']:
# self.userError('permission denied')
# return
if id and p == 'start':
# TODO: Check the status of the session in DB
if id in self.threads:
self.userError('this session has already started')
return
else:
data = {
'slice_id': id,
'status': 'started',
'start_date': format_date()
}
# Create a session in DB
session = yield r.db(s.db.name).table('sessions').insert(
data, conflict='update').run(self.dbconnection)
session_id = session['generated_keys'][0]
# Add the session to the threads
self.threads[id] = []
# TODO: F-Interop Orchestrator Integration
# send a call to the F-Interop Orchestrator
# XXX Meanwhile start the FakeInterop session
yield self.startFakeInterop(session_id, id)
# listen to the session
logger.info('Listen to the session %s' % session_id)
yield self.listenSession(session_id, id)
elif id and p == 'stop':
# TODO: Check the status of the session in DB
# Get session in DB
data = yield self.getSession(id)
if data:
data['status'] = 'stopped'
data['end_date'] = format_date()
# Update session in DB
r.db(s.db.name).table('sessions').get(
data['id']).update(data).run(self.dbconnection)
if id in self.threads:
# TODO: send a call to the F-Interop Orchestrator
# XXX Meanwhile stop FakeInterop process
# Stop Listening process
for t in self.threads[id]:
t.terminate()
del self.threads[id]
logger.info('Stop listening to the session')
stopSession(id)
else:
self.userError('this session is not running')
return
else:
self.userError('not supported by the REST API')
return
# TODO: send back the result success with data about the session
self.write(
json.dumps(
{
"result": "success",
"data": [],
"error": None,
"debug": None
},
cls=myJSONEncoder))
def message(self, from_user, message=None):
self['messages'].append({
'from': from_user,
'message': message,
'timestamp': format_date(),
})
def updated(self, value=None):
if value:
self['updated'] = value
else:
self['updated'] = format_date()
def __init__(self, event):
self['messages'] = event.get('messages', [])
self['log'] = event.get('log', [])
self['manager'] = event.get('manager', None)
##
# Default status when creating the event is NEW
self.previous_status = event.get('previous_status', EventStatus.INIT)
self.status = event.get('status', EventStatus.NEW)
try:
self.action = event['action']
except KeyError:
raise Exception("Event action not specified")
##
# If ID is present this is a previously
# created event, or we don't set it
try:
self.id = event['id']
except KeyError:
pass
##
# User making the request
#
try:
self.user = event['user']
except KeyError:
raise Exception("User Id not specified")
##
# Object of the event
#
try:
self.object = event['object']
except KeyError:
raise Exception("Event Object not specified")
except Exception as e:
raise Exception("{0}".format(e))
##
# data is a dictionary of changes for the object
#
if 'data' in event:
self.data = event['data']
else:
self.data = {}
#try:
# self.data = event['data']
#except KeyError as e:
# import traceback
# traceback.print_exc()
# self.data = {}
try:
self['created'] = format_date(event['created'])
except KeyError:
self['created'] = format_date()
pass
try:
self.updated(format_date(event['updated']))
except KeyError:
pass
##
# Notify the user
try:
self.notify = event['notify']
except KeyError:
self.notify = False
pass
def syncSlices(id=None):
with lock:
logger.info("Worker slices starting synchronization")
try:
# DB connection
dbconnection = db.connect()
# Update an existing Slice
if id:
slices = Slices([Slice(db.get(dbconnection, table='slices', id=id))])
# MySliceLib Query Slices
else:
slices = q(Slice).get()
if len(slices)==0:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
# ------------------------------------------------------
# Synchronize resources of a Slice at AMs
# !!! only if the slice_id is specified !!!
# Otherwise it is way too long to synchronize all slices
# ------------------------------------------------------
# TODO: trigger this function in background for a user
# that want to refresh his slice / when he selected one
# ------------------------------------------------------
if id:
for slice in slices:
if len(slice.users) > 0:
try:
u = User(db.get(dbconnection, table='users', id=slice.users[0]))
logger.info("Synchronize slice %s:" % slice.hrn)
# Synchronize resources of the slice only if we have the user's private key or its credentials
# XXX Should use delegated Credentials
#if (hasattr(u,'private_key') and u.private_key is not None and len(u.private_key)>0) or (hasattr(u,'credentials') and len(u.credentials)>0):
if u.private_key or (hasattr(u,'credentials') and len(u.credentials)>0):
user_setup = UserSetup(u,myslicelibsetup.endpoints)
logger.info("Slice.id(%s).get() with user creds" % slice.hrn)
s = q(Slice, user_setup).id(slice.id).get().first()
db.slices(dbconnection, s.dict(), slice.id)
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with slice %s" % slice.id)
logger.exception(str(e))
raise
else:
logger.info("slice %s has no users" % slice.hrn)
# update local slice table
else:
if len(slices)>0:
local_slices = db.slices(dbconnection)
# Add slices from Registry unkown from local DB
for s in slices:
if not db.get(dbconnection, table='slices', id=s.id):
logger.info("Found new slice from Registry: %s" % s.id)
db.slices(dbconnection, s.dict(), s.id)
# Update slices known in local DB
for ls in local_slices :
logger.info("Synchronize Slice {}".format(ls['id']))
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
if not slices.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete slices that have been deleted elsewhere
db.delete(dbconnection, 'slices', ls['id'])
logger.info("Slice {} deleted".format(ls['id']))
else:
db.slices(dbconnection, ls, ls['id'])
else:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(str(e))
raise
logger.info("Worker slices finished period synchronization")
def save(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).save(setup)
errors = result['errors']
result = {**(self.dict()), **result['data'][0]}
if not errors:
for r in result['resources']:
if (not 'services' in r) or (not r['services']):
logger.warning("result from slice.save didn't had login info")
logger.warning("sleeping 10s before asking again to AMs")
import time
time.sleep(10)
slice = q(Slice, setup).id(self.id).get().first()
result = slice.dict()
break
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Slice created
if current is None:
db.slices(dbconnection, result)
current = db.get(dbconnection, table='slices', id=self.id)
# Update existing slice
else:
db.slices(dbconnection, result, self.id)
# Update users both previously and currently in Slice
users = list(set(current['users']) | set(self.getAttribute('users')))
for u in users:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice save()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.save(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
if u in current['users'] and u not in self.getAttribute('users'):
# Remove slice from user
del u['slices'][self.id]
elif u not in current['users'] and u in self.getAttribute('users'):
# Add slice to user
u['slice'].append(self.id)
db.users(dbconnection, user, u)
# Update the Project of the slice
logger.debug("cooko slice: {}".format(self))
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = project['slices'] + [self.id]
db.projects(dbconnection, project)
# Insert / Delete Leases if necessary
if self.hasLeases:
flag = -1
for lease in self.leases:
# No resources reserved
if len(result['leases'])==0:
flag = -1
# All resources of a Lease have been succesfully reserved
elif lease['resources'] == result['leases'][0]['resources']:
flag = 0
# Some Resources of a Lease have been reserved
elif len(set(lease['resources']).intersection(set(result['leases'][0]['resources']))) > 0:
db.leases(dbconnection, lease)
flag = 1
for lease in self.removedLeases:
if lease not in result['leases']:
db.delete(dbconnection, 'leases', lease.id)
flag = False
if flag == -1:
errors.append("No reservation has been accepted by the testbeds")
elif flag == 1:
errors.append("Some resources have been reserved others were unavailable")
raise SliceWarningException(errors)
if errors:
raise SliceException(errors)
else:
return True
def syncUsers(lock, email=None):
# DB connection
dbconnection = db.connect()
# acquires lock
with lock:
logger.info("Worker users starting synchronization")
try:
if email:
users = q(User).filter('email', email).get()
else:
users = q(User).get()
"""
update local user table
"""
if len(users) > 0:
# Add users from Registry unkown from local DB
# this is used to bootstrap with init_user script
for u in users:
logger.debug("looking for {} in local db".format(u.id))
if not db.get(dbconnection, table='users', id=u.id):
local_users = db.get(dbconnection, table='users')
logger.warning("Number of users in local db: %s" %
len(local_users))
#print("this user is not in local db, add it")
logger.info("Found new user from Registry: %s" % u.id)
#logger.info("We don't add the missing users yet, as portal is the single point of entry")
db.users(dbconnection, u.dict())
local_users = db.get(dbconnection, table='users')
# Update users known in local DB
for lu in local_users:
logger.info("Synchronize user %s" % lu['id'])
try:
# add status if not present and update on db
if not 'status' in lu:
lu['status'] = Status.ENABLED
lu['enabled'] = format_date()
if not users.has(lu['id']) and lu[
'status'] is not Status.PENDING:
# delete user that has been deleted in Reg
db.delete(dbconnection, 'users', lu['id'])
logger.info("User {} deleted".format(lu['id']))
else:
remote_user = next(
(item
for item in users if item.id == lu['id']),
False)
if remote_user:
# merge fields of local user with remote
# keep local values for
# password, private_key, public_key and generate_keys
updated_user = remote_user.merge(dbconnection)
updated_user = updated_user.dict()
# if user has private key
# update its Credentials
if 'private_key' in updated_user and updated_user[
'private_key'] is not None:
updated_user = update_credentials(
dbconnection, updated_user)
# Update user
#logger.debug("Update user %s" % updated_user['id'])
db.users(dbconnection, updated_user,
updated_user['id'])
except Exception as e:
logger.warning("Could not synchronize user %s" %
lu['id'])
logger.exception(e)
raise
else:
logger.warning(
"Query users is empty, check myslicelib and the connection with SFA Registry"
)
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(e)
raise
logger.info("Worker users finished period synchronization")
def save(self, dbconnection, setup=None):
# Get Authority from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='authorities', id=self.id)
new_users = self.handleDict('users')
new_pi_users = self.handleDict('pi_users')
result = super(Authority, self).save(setup)
errors = result['errors']
if errors:
raise AuthorityException(errors)
logger.debug(result)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Authority created
if current is None:
db.authorities(dbconnection, result)
current = db.get(dbconnection, table='authorities', id=self.id)
# Update existing authority
else:
db.authorities(dbconnection, result, self.id)
# Create new users under a New Authority
# Otherwise users are created with User.save()
for u in new_users:
if isinstance(u, dict):
if not "email" in u:
raise Warning("user has no email can not be created")
user = db.get(dbconnection,
table='users',
filter={'email': u['email']},
limit=1)
# If the user doesn't exit, create it
if not user:
user = User(u)
user.setAttribute('authority', self.id)
userCreated = user.save(dbconnection)
if not userCreated:
raise Warning("user has not been created")
self.users.append(user.id)
modified = True
# Grant or Revoke PI Rights
current['pi_users'] = current.get('pi_users', [])
pi_users = current['pi_users'] + self.getAttribute(
'pi_users') + new_pi_users
modified = False
for u in pi_users:
if isinstance(u, dict):
if not "email" in u:
raise Warning(
"pi_user has no email can not be added as PI")
user = db.get(dbconnection,
table='users',
filter={'email': u['email']},
limit=1)
if not user:
raise Warning("user does not exist, can't be PI")
else:
user = User(user[0])
else:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
# REMOVE PI Rights
if user.id not in self.getAttribute('pi_users') and not any(
d['email'] == user.email for d in new_pi_users):
self.removePi(user)
modified = True
# GRANT PI Rights
elif user.id not in current['pi_users']:
self.addPi(user)
modified = True
logger.debug("Update user %s in Authority save()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
if modified:
result = super(Authority, self).save(setup)
errors = result['errors']
if errors:
raise AuthorityException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
db.authorities(dbconnection, result, self.id)
return True
