def main(argv):
try:
if len(argv) != 1:
print("Help: use the command with one of the parameters")
print("clean.py all|authorities|projects|users|slices")
sys.exit(2)
if argv[0].startswith('auth') or argv[0] == 'all':
print("clean authorities...")
objects = q(Authority).get()
cleanRegistry(objects)
if argv[0].startswith('p') or argv[0] == 'all':
print("clean projects...")
objects = q(Project).get()
cleanRegistry(objects)
if argv[0].startswith('u') or argv[0] == 'all':
print("clean users...")
objects = q(User).get()
cleanRegistry(objects)
if argv[0].startswith('s') or argv[0] == 'all':
print("clean slices...")
objects = q(Slice).get()
cleanRegistry(objects)
except Exception as e:
import traceback
traceback.print_exc()
print("Help: use the command with one of the parameters")
print("clean.py all|authorities|projects|users|slices")
sys.exit(2)
def main(argv):
try:
if len(argv) != 2:
print(
"Help: use the command with one of the parameters and server name"
)
print(
"clean.py all|authorities|projects|users|slices <server.fqdn.or.ip>"
)
print("EXAMPLE: clean.py all zeus.noc.onelab.eu")
sys.exit(2)
if argv[0].startswith('auth') or argv[0] == 'all':
print("clean authorities...")
clean('authorities', argv[1])
# clean(cookies, 'authorities', argv[1])
objects = q(Authority).get()
cleanRegistry(objects)
if argv[0].startswith('p') or argv[0] == 'all':
print("clean projects...")
clean('projects', argv[1])
# clean(cookies, 'projects', argv[1])
objects = q(Project).get()
cleanRegistry(objects)
if argv[0].startswith('u') or argv[0] == 'all':
print("clean users...")
clean('users', argv[1])
# clean(cookies, 'users', argv[1])
objects = q(User).get()
cleanRegistry(objects)
if argv[0].startswith('s') or argv[0] == 'all':
print("clean slices...")
clean('slices', argv[1])
# clean(cookies, 'slices', argv[1])
objects = q(Slice).get()
cleanRegistry(objects)
except Exception as e:
import traceback
traceback.print_exc()
print(
"Help: use the command with one of the parameters and server name")
print(
"clean.py all|authorities|projects|users|slices <server.fqdn.or.ip>"
)
print("EXAMPLE: clean.py all zeus.noc.onelab.eu")
sys.exit(2)
def delete(self, dbconnection, setup=None):
# Get Project from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='projects', id=self.id)
result = super(Project, self).delete(setup)
errors = result['errors']
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise ProjectException(errors)
db.delete(dbconnection, 'projects', self.id)
for u in current['pi_users']:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
logger.debug("Update user %s after Project delete()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
# Slices will be removed by Sync
return True
def syncProjects(lock):
# DB connection
dbconnection = connect()
# acquires lock
with lock:
logger.info("Worker projects starting synchronization")
# MySliceLib Query Slices
p = q(Project).get()
# update local projects table
if len(p)>0:
lprojects = db.projects(dbconnection, p.dict())
for ls in lprojects :
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
db.projects(dbconnection, ls)
if not p.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete projects that have been deleted elsewhere
db.delete(dbconnection, 'projects', ls['id'])
logger.info("Project {} deleted".format(ls['id']))
else:
logger.warning("Query projects is empty, check myslicelib and the connection with SFA Registry")
def save(self, dbconnection, setup=None):
# Get Project from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='projects', id=self.id)
result = super(Project, self).save(setup)
errors = result['errors']
if errors:
raise ProjectException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Project created
if current is None:
db.projects(dbconnection, result)
current = db.get(dbconnection, table='projects', id=self.id)
# Update existing project
else:
db.projects(dbconnection, result, self.id)
# update pi_users after Save
pi_users = list(
set(current['pi_users']) | set(self.getAttribute('pi_users')))
for u in pi_users:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
logger.debug("Update user %s after Project save()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
# update slices after Save
slices = list(
set(current['slices']) | set(self.getAttribute('slices')))
if setup:
setup.setEndpoints(myslicelibsetup.endpoints)
for s in current['slices']:
sl = q(Slice, setup).id(s).get().first()
db.slices(dbconnection, sl.dict())
return True
def run():
"""
A thread that will check resource availability and information
"""
logger.info("Agent endpoints starting")
"""
DB connection
"""
dbconnection = db.connect()
"""
MySliceLib Query Testbeds
"""
testbeds = q(Testbed).get()
logger.info(testbeds.dict())
def delete(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).delete(setup)
errors = result['errors']
# Signal only Registry Errors
if errors:
raising = False
for err in errors:
if err['type'] == "Reg":
if "Record not found" in err['exception']:
raising = False
else:
raising = True
if raising:
raise SliceException(errors)
db.delete(dbconnection, 'slices', self.id)
for u in current['users']:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice delete()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.delete(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
# Remove slice from user
del u['slices'][self.id]
db.users(dbconnection, user, u)
# Update the Project of the slice
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = list(set(project['slices']) - set([self.id]))
db.projects(dbconnection, project)
# Warning if errors on AMs
#if errors:
# raise SliceWarningException(errors)
return True
def syncTestbeds():
try:
# retreive version and status info from the testbeds
t = q(Testbed).version()
if len(t) > 0:
# syncs testbeds configured with the db
db.syncTestbeds(t)
else:
logger.warning(
"Check myslicelib and SFA, q(Testbed).version() returned empty"
)
except Exception as e:
logger.exception("Service does not seem to be available")
raise
def syncResources():
try:
# retreive resources from testbeds
r = q(Resource).get()
if len(r)>0:
# syncs resources configured with the db
db.syncResources(r)
else:
logger.warning("Check myslicelib and SFA, q(Resource).get() returned empty")
except Exception as e:
import traceback
traceback.print_exc()
logger.exception("Service does not seem to be available")
raise
def syncLeases():
try:
logger.debug("Query Lease")
ll = q(Lease).get()
logger.debug("syncLeases")
# syncs leases configured with the db
slices = db.syncLeases(ll)
for s in slices:
logger.info("Synchronize slice %s after syncLeases" % s)
syncSlices(s)
except Exception as e:
#import traceback
#traceback.print_exc()
logger.exception("Service does not seem to be available")
logger.exception(str(e))
raise
def delete(self, dbconnection, setup=None):
logger.debug("Delete Authority %s" % self.id)
# Get Authority from local DB
# to update the pi_users after Save
logger.debug("Get current object")
current = db.get(dbconnection, table='authorities', id=self.id)
logger.debug("Delete sent to myslicelib")
result = super(Authority, self).delete(setup)
logger.debug("result from myslicelib")
logger.debug(result)
errors = result['errors']
logger.debug("checking errors")
if errors:
raising = True
for err in errors:
if "Record not found" in err['exception']:
raising = False
break
if raising:
raise AuthorityException(errors)
logger.debug("Delete Authority from local DB")
db.delete(dbconnection, 'authorities', self.id)
logger.debug("Delete users of the Authority from local DB")
for u in current['users']:
logger.debug("Delete user %s" % u)
db.delete(dbconnection, 'users', u)
logger.debug("Update PI users of the Authority in local DB")
for u in current['pi_users']:
logger.debug("Get user %s" % u)
user = q(User).id(u).get().first()
if user:
logger.debug("Update user %s in Authority delete()" % u)
logger.debug(user)
user = user.merge(dbconnection)
db.users(dbconnection, user.dict(), user.id)
return True
def syncSlices(id=None):
with lock:
logger.info("Worker slices starting synchronization")
try:
# DB connection
dbconnection = db.connect()
# Update an existing Slice
if id:
slices = Slices([Slice(db.get(dbconnection, table='slices', id=id))])
# MySliceLib Query Slices
else:
slices = q(Slice).get()
if len(slices)==0:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
# ------------------------------------------------------
# Synchronize resources of a Slice at AMs
# !!! only if the slice_id is specified !!!
# Otherwise it is way too long to synchronize all slices
# ------------------------------------------------------
# TODO: trigger this function in background for a user
# that want to refresh his slice / when he selected one
# ------------------------------------------------------
if id:
for slice in slices:
if len(slice.users) > 0:
try:
u = User(db.get(dbconnection, table='users', id=slice.users[0]))
logger.info("Synchronize slice %s:" % slice.hrn)
# Synchronize resources of the slice only if we have the user's private key or its credentials
# XXX Should use delegated Credentials
#if (hasattr(u,'private_key') and u.private_key is not None and len(u.private_key)>0) or (hasattr(u,'credentials') and len(u.credentials)>0):
if u.private_key or (hasattr(u,'credentials') and len(u.credentials)>0):
user_setup = UserSetup(u,myslicelibsetup.endpoints)
logger.info("Slice.id(%s).get() with user creds" % slice.hrn)
s = q(Slice, user_setup).id(slice.id).get().first()
db.slices(dbconnection, s.dict(), slice.id)
except Exception as e:
import traceback
traceback.print_exc()
logger.error("Problem with slice %s" % slice.id)
logger.exception(str(e))
raise
else:
logger.info("slice %s has no users" % slice.hrn)
# update local slice table
else:
if len(slices)>0:
local_slices = db.slices(dbconnection)
# Add slices from Registry unkown from local DB
for s in slices:
if not db.get(dbconnection, table='slices', id=s.id):
logger.info("Found new slice from Registry: %s" % s.id)
db.slices(dbconnection, s.dict(), s.id)
# Update slices known in local DB
for ls in local_slices :
logger.info("Synchronize Slice {}".format(ls['id']))
# add status if not present and update on db
if not 'status' in ls:
ls['status'] = Status.ENABLED
ls['enabled'] = format_date()
if not slices.has(ls['id']) and ls['status'] is not Status.PENDING:
# delete slices that have been deleted elsewhere
db.delete(dbconnection, 'slices', ls['id'])
logger.info("Slice {} deleted".format(ls['id']))
else:
db.slices(dbconnection, ls, ls['id'])
else:
logger.warning("Query slices is empty, check myslicelib and the connection with SFA Registry")
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(str(e))
raise
logger.info("Worker slices finished period synchronization")
def save(self, dbconnection, setup=None):
# Get Slice from local DB
# to update the users after Save
current = db.get(dbconnection, table='slices', id=self.id)
result = super(Slice, self).save(setup)
errors = result['errors']
result = {**(self.dict()), **result['data'][0]}
if not errors:
for r in result['resources']:
if (not 'services' in r) or (not r['services']):
logger.warning("result from slice.save didn't had login info")
logger.warning("sleeping 10s before asking again to AMs")
import time
time.sleep(10)
slice = q(Slice, setup).id(self.id).get().first()
result = slice.dict()
break
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Slice created
if current is None:
db.slices(dbconnection, result)
current = db.get(dbconnection, table='slices', id=self.id)
# Update existing slice
else:
db.slices(dbconnection, result, self.id)
# Update users both previously and currently in Slice
users = list(set(current['users']) | set(self.getAttribute('users')))
for u in users:
user = q(User).id(u).get().first()
if user:
user = user.merge(dbconnection)
logger.debug("Update user %s after Slice save()" % u)
logger.debug(user)
user = user.dict()
else:
logger.error("Could not update user after Slice.save(), no answer from Registry")
logger.warning("Updating the local DB manually")
user = db.get(dbconnection, table='users', id=u)
if u in current['users'] and u not in self.getAttribute('users'):
# Remove slice from user
del u['slices'][self.id]
elif u not in current['users'] and u in self.getAttribute('users'):
# Add slice to user
u['slice'].append(self.id)
db.users(dbconnection, user, u)
# Update the Project of the slice
logger.debug("cooko slice: {}".format(self))
project = db.get(dbconnection, table='projects', id=self.project)
project['slices'] = project['slices'] + [self.id]
db.projects(dbconnection, project)
# Insert / Delete Leases if necessary
if self.hasLeases:
flag = -1
for lease in self.leases:
# No resources reserved
if len(result['leases'])==0:
flag = -1
# All resources of a Lease have been succesfully reserved
elif lease['resources'] == result['leases'][0]['resources']:
flag = 0
# Some Resources of a Lease have been reserved
elif len(set(lease['resources']).intersection(set(result['leases'][0]['resources']))) > 0:
db.leases(dbconnection, lease)
flag = 1
for lease in self.removedLeases:
if lease not in result['leases']:
db.delete(dbconnection, 'leases', lease.id)
flag = False
if flag == -1:
errors.append("No reservation has been accepted by the testbeds")
elif flag == 1:
errors.append("Some resources have been reserved others were unavailable")
raise SliceWarningException(errors)
if errors:
raise SliceException(errors)
else:
return True
def syncUsers(lock, email=None):
# DB connection
dbconnection = db.connect()
# acquires lock
with lock:
logger.info("Worker users starting synchronization")
try:
if email:
users = q(User).filter('email', email).get()
else:
users = q(User).get()
"""
update local user table
"""
if len(users) > 0:
# Add users from Registry unkown from local DB
# this is used to bootstrap with init_user script
for u in users:
logger.debug("looking for {} in local db".format(u.id))
if not db.get(dbconnection, table='users', id=u.id):
local_users = db.get(dbconnection, table='users')
logger.warning("Number of users in local db: %s" %
len(local_users))
#print("this user is not in local db, add it")
logger.info("Found new user from Registry: %s" % u.id)
#logger.info("We don't add the missing users yet, as portal is the single point of entry")
db.users(dbconnection, u.dict())
local_users = db.get(dbconnection, table='users')
# Update users known in local DB
for lu in local_users:
logger.info("Synchronize user %s" % lu['id'])
try:
# add status if not present and update on db
if not 'status' in lu:
lu['status'] = Status.ENABLED
lu['enabled'] = format_date()
if not users.has(lu['id']) and lu[
'status'] is not Status.PENDING:
# delete user that has been deleted in Reg
db.delete(dbconnection, 'users', lu['id'])
logger.info("User {} deleted".format(lu['id']))
else:
remote_user = next(
(item
for item in users if item.id == lu['id']),
False)
if remote_user:
# merge fields of local user with remote
# keep local values for
# password, private_key, public_key and generate_keys
updated_user = remote_user.merge(dbconnection)
updated_user = updated_user.dict()
# if user has private key
# update its Credentials
if 'private_key' in updated_user and updated_user[
'private_key'] is not None:
updated_user = update_credentials(
dbconnection, updated_user)
# Update user
#logger.debug("Update user %s" % updated_user['id'])
db.users(dbconnection, updated_user,
updated_user['id'])
except Exception as e:
logger.warning("Could not synchronize user %s" %
lu['id'])
logger.exception(e)
raise
else:
logger.warning(
"Query users is empty, check myslicelib and the connection with SFA Registry"
)
except Exception as e:
import traceback
traceback.print_exc()
logger.exception(e)
raise
logger.info("Worker users finished period synchronization")
def save(self, dbconnection, setup=None):
# Get Authority from local DB
# to update the pi_users after Save
current = db.get(dbconnection, table='authorities', id=self.id)
new_users = self.handleDict('users')
new_pi_users = self.handleDict('pi_users')
result = super(Authority, self).save(setup)
errors = result['errors']
if errors:
raise AuthorityException(errors)
logger.debug(result)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
# New Authority created
if current is None:
db.authorities(dbconnection, result)
current = db.get(dbconnection, table='authorities', id=self.id)
# Update existing authority
else:
db.authorities(dbconnection, result, self.id)
# Create new users under a New Authority
# Otherwise users are created with User.save()
for u in new_users:
if isinstance(u, dict):
if not "email" in u:
raise Warning("user has no email can not be created")
user = db.get(dbconnection,
table='users',
filter={'email': u['email']},
limit=1)
# If the user doesn't exit, create it
if not user:
user = User(u)
user.setAttribute('authority', self.id)
userCreated = user.save(dbconnection)
if not userCreated:
raise Warning("user has not been created")
self.users.append(user.id)
modified = True
# Grant or Revoke PI Rights
current['pi_users'] = current.get('pi_users', [])
pi_users = current['pi_users'] + self.getAttribute(
'pi_users') + new_pi_users
modified = False
for u in pi_users:
if isinstance(u, dict):
if not "email" in u:
raise Warning(
"pi_user has no email can not be added as PI")
user = db.get(dbconnection,
table='users',
filter={'email': u['email']},
limit=1)
if not user:
raise Warning("user does not exist, can't be PI")
else:
user = User(user[0])
else:
user = q(User).id(u).get().first()
user = user.merge(dbconnection)
# REMOVE PI Rights
if user.id not in self.getAttribute('pi_users') and not any(
d['email'] == user.email for d in new_pi_users):
self.removePi(user)
modified = True
# GRANT PI Rights
elif user.id not in current['pi_users']:
self.addPi(user)
modified = True
logger.debug("Update user %s in Authority save()" % u)
logger.debug(user)
db.users(dbconnection, user.dict(), user.id)
if modified:
result = super(Authority, self).save(setup)
errors = result['errors']
if errors:
raise AuthorityException(errors)
result = {**(self.dict()), **result['data'][0]}
# add status if not present and update on db
if not 'status' in result:
result['status'] = Status.ENABLED
result['enabled'] = format_date()
db.authorities(dbconnection, result, self.id)
return True
def main(argv):
try:
opts, args = getopt.getopt(
argv, "he:P:k:p:s",
["email=", "password="******"private_key=", "public_key=", "sync="])
except getopt.GetoptError:
print(
'init_user.py -e <email> -P <password> -k <private_key path> -p <public_key path> -s <synchronize with Registry>'
)
sys.exit(2)
if (len(opts) < 4):
print('Missing parameters:')
print(
'init_user.py -e <email> -P <password> -k <private_key path> -p <public_key path> -s <synchronize with Registry>'
)
sys.exit(2)
sync = False
for opt, arg in opts:
if opt == '-h':
print(
'init_user.py -e <email> -P <password> -k <private_key path> -p <public_key path> -s <synchronize with Registry>'
)
sys.exit()
elif opt in ("-e", "--email"):
email = arg
elif opt in ("-P", "--password"):
password = crypt_password(arg)
elif opt in ("-k", "--private_key"):
f = open(arg, 'r')
private_key = f.read()
elif opt in ("-p", "--public_key"):
f = open(arg, 'r')
public_key = f.read()
elif opt in ("-s", "--sync"):
if arg.lower() in ["true", "yes", "y"]:
sync = True
dbconnection = db.connect()
lock = threading.Lock()
# Synchronize the users from SFA Registry into the DB
if sync:
print("sync user %s" % email)
syncUsers(lock, email=email, job=False)
else:
try:
# Get the user from SFA Registry
print("get user %s from SFA Reg" % email)
remote_user = q(User).filter('email', email).get().first()
pprint(remote_user)
# merge fields from script with remote_user
remote_user.setAttribute('password', password)
remote_user.setAttribute('private_key', private_key)
remote_user.setAttribute('generate_keys', False)
remote_user.setAttribute('public_key', public_key)
remote_user.setAttribute('keys', [public_key])
r.db('myslice').table('users').insert(
remote_user.dict()).run(dbconnection)
result = r.db('myslice').table('users').get(
remote_user.id).run(dbconnection)
#result = remote_user.save(dbconnection)
if result:
print("User saved")
else:
print("Error during save")
pprint(result)
# if user has private key
# update its Credentials
#if 'private_key' in updated_user:
# updated_user = update_credentials(updated_user)
except Exception as e:
import traceback
traceback.print_exc()