def test_project_display(project_id):
feed_utils.heading("Trying project display. project_id: %s" % project_id)
project_info = project.get_project(project_id)
if not project_info:
feed_utils.failed("failed getting project details")
else:
feed_utils.ok("project details: %s" % project_info)
def test_ssh(nels_id):
feed_utils.heading("ssh use case. nels_id: %s" % nels_id)
credentail = ssh_utils.get_ssh_credentials(nels_id)
if not credentail:
feed_utils.failed("fetching key")
return False
feed_utils.ok("ssh key fetched")
(host, username, key_file) = credentail
(status, items,
error) = run_utils.launch_remote_with_key(key_file, username, host, "ls ")
if status != 0:
feed_utils.error(error)
return False
if "Personal" in items:
feed_utils.ok("Personal folder found")
else:
feed_utils.failed("Personal folder not found")
if "Projects" in items:
feed_utils.ok("Projects folder found")
else:
feed_utils.failed("Projects folder not found")
feed_utils.info("cleaning key file")
run_utils.launch_cmd("rm -f %s" % key_file)
return True
def print_config():
feed_utils.heading("Configurations")
feed_utils.info("port: %s" % PORT)
feed_utils.info("implicit clients")
feed_utils.push_in()
for client in IMPLICIT_CLIENTS:
feed_utils.info(client)
feed_utils.push_out()
def test_user_registration(idp_number, federated_id, name, email, user_type_id, is_active, affiliation):
feed_utils.heading("Trying user registration")
new_id = users.register_user(idp_number, federated_id, name, email, user_type_id, is_active, affiliation)
if not new_id:
feed_utils.failed("user registration failed")
else:
feed_utils.ok("new user registered. nels_id: %s " % new_id)
test_users_list.test_user_display(new_id)
def test_project_add(name, description):
feed_utils.heading("Trying project creation")
new_id = project.add_project(name, description)
if not new_id:
feed_utils.failed("project creation failed")
else:
feed_utils.ok("new created prject. project_id: %s " % new_id)
test_project_display(new_id)
def test_quota_del(id):
feed_utils.heading("Trying quota deletion. id : %s" % id)
if not sbi_quotas.delete_quota(id):
feed_utils.failed("deletion of quota failed")
return False
feed_utils.ok("deletion of quota")
return True
def test_project_display_dataset(project_id, dataset_id):
feed_utils.heading(
"Trying dataset in project display. project_id: %s, dataset_id %s" %
(project_id, dataset_id))
project_info = sbi_project.get_dataset_in_project(project_id, dataset_id)
if not project_info:
feed_utils.failed("failed getting dataset details")
else:
feed_utils.ok("dataset details: %s" % project_info)
def test_recompute_disc_usage():
feed_utils.heading("Trying recompute sbi project disc usage")
success = sbi_project.recompute_project_disc_usage()
if success is False:
feed_utils.failed("recompute failed")
else:
feed_utils.ok("recompute success")
return success
def test_project_add_dataset(project_id, federated_id):
feed_utils.heading("Trying adding dataset to project")
success = sbi_project.add_dataset_to_project(project_id, federated_id)
if success != True:
feed_utils.failed("adding dataset to project failed")
else:
feed_utils.ok("new dataset in project. ")
return project_id
def test_move(nels_id):
feed_utils.heading("move use case. nels_id: %s" % nels_id)
credentail = ssh_utils.get_ssh_credentials(nels_id)
if not credentail:
feed_utils.failed("fetching key")
return False
feed_utils.ok("ssh key fetched")
(host, username, key_file) = credentail
src_file = "Personal/dummy.txt"
dest_dir = "Personal/dummy"
if not ssh_utils.write_dummy_file(key_file, username, host, src_file,
"5M"):
feed_utils.failed("write test file")
return False
feed_utils.ok("write test file")
if not ssh_utils.create_folder(key_file, username, host, dest_dir):
feed_utils.failed("create test directory")
return False
feed_utils.ok("create test directory")
job_id = job.add_job(job.STORAGE_MOVE, nels_id, [src_file], dest_dir)
if job_id == -1:
feed_utils.failed("job submission")
return False
feed_utils.ok("job submission. job-id: %s" % job_id)
job.wait_for_job(job_id)
[exit_code, output,
error] = run_utils.launch_remote_with_key(key_file, username, host,
"ls %s" % dest_dir)
if exit_code != 0:
feed_utils.error(error)
return False
if "dummy.txt" not in output:
feed_utils.failed("verify moved file")
return False
feed_utils.ok("verify moved file")
#validate file
[exit_code, output,
error] = run_utils.launch_remote_with_key(key_file, username, host,
"rm -rf %s" % (dest_dir))
if not exit_code == 0:
feed_utils.error(error)
return False
feed_utils.ok("cleanup")
#clean up
if not job.delete_job(job_id):
feed_utils.failed("job delete")
return False
feed_utils.ok("job delete")
return True
def test_project_delete_member(project_id, federated_id):
feed_utils.heading("Trying delete member from project")
success = sbi_project.delete_user_from_project(project_id, federated_id)
if success != True:
feed_utils.failed("delete member from project failed")
else:
feed_utils.ok("removed member in project. ")
return project_id
def test_quota_add(name, federated_id):
feed_utils.heading("Trying quota creation")
created_code = sbi_quotas.add_quota(name, federated_id)
if created_code == None:
feed_utils.failed("quota creation failed")
return False
feed_utils.ok("new created quota ")
return True
def test_project_delete_dataset(project_id, dataset_id):
feed_utils.heading("Trying deleting dataset from project")
success = sbi_project.delete_dataset_in_project(project_id, dataset_id)
if success != True:
feed_utils.failed("deleting dataset in project failed")
else:
feed_utils.ok("deleted dataset in project. ")
return project_id
def test_project_add(quota_id, name, federated_id):
feed_utils.heading("Trying project creation")
project_id = sbi_project.add_project(quota_id, name, federated_id)
if project_id is None:
feed_utils.failed("project creation failed")
else:
feed_utils.ok("new created project. project id: %s " % project_id)
return project_id
def test_project_add_member(project_id, membership_type, federated_id):
feed_utils.heading("Trying adding member to project")
success = sbi_project.add_user_to_project(project_id, membership_type,
federated_id)
if success != True:
feed_utils.failed("adding member to project failed")
else:
feed_utils.ok("new member in project. ")
return project_id
def test_projects_list():
feed_utils.heading("Trying projects list")
project_ids = project.get_project_ids()
if not project_ids:
feed_utils.failed("get project ids")
return False
feed_utils.ok("fetched %s project ids" % len(project_ids))
random_project_id = project_ids[random.randrange(0, len(project_ids) - 1)]
test_project_display(random_project_id)
return True
def test_copy_nels_to_sbi(nels_id):
feed_utils.heading("copy use case nels to sbi. nels_id: %s" % nels_id)
credentail = ssh_utils.get_ssh_credentials(nels_id)
if not credentail:
feed_utils.failed("fetching key")
return False
feed_utils.ok("ssh key fetched")
(host, username, key_file) = credentail
test_file = "dummy5.txt"
src_dir = "Personal"
src_file = "%s/%s" % (src_dir, test_file)
dest_dir = ''
if not ssh_utils.write_dummy_file(key_file, username, host, src_file,
"5k"): # 5M
feed_utils.failed("write test file")
return False
feed_utils.ok("write test file")
dataset_id = "c16ff090-605d-4228-a8c9-3713a9a40e45"
dataset_name = "regtest1feb"
subtype = "Intensities"
subtype_id = 1124874
job_id = job.add_sbi_job(job.SBI_PULL, nels_id, [test_file], [], host,
username, file_utils.read_file_content(key_file),
dataset_id, dataset_name, subtype, subtype_id,
src_dir, "")
if job_id == -1:
feed_utils.failed("job submission")
return False
feed_utils.ok("job submission. job-id: %s" % job_id)
job.wait_for_job(job_id)
# validate file (to do)
# clean up
if not job.delete_job(job_id):
feed_utils.failed("job delete")
return False
feed_utils.ok("job delete")
# clean up on NeLS side
[exit_code, output,
error] = run_utils.launch_remote_with_key(key_file, username, host,
"rm -rf %s " % (src_file))
if exit_code != 0:
feed_utils.error(error)
return False
feed_utils.ok("clean up")
return True
def test_users_list():
feed_utils.heading("Trying sbi user ids")
user_ids = sbi_users.get_user_ids()
if not user_ids:
feed_utils.failed("get user ids")
return
for user_id in user_ids:
feed_utils.info(user_id)
feed_utils.ok("")
def test_project_list_all_datasets(project_id):
feed_utils.heading("Trying list datasets in project. project_id: %s" %
project_id)
project_info = sbi_project.get_datasets_in_project(project_id)
if project_info == None:
feed_utils.failed("failed getting project datasets")
elif project_info == []:
feed_utils.ok("project without datasets: %s" % project_info)
else:
feed_utils.ok("project datasets: %s" % project_info)
def test_project_list_all_subtypes(project_id, dataset_id):
feed_utils.heading(
"Trying list subtypes of a dataset in project. project_id: %s, dataset_id %s"
% (project_id, dataset_id))
project_info = sbi_project.get_subtypes_in_dataset_in_project(
project_id, dataset_id)
if project_info == None:
feed_utils.failed("failed getting project dataset subtypes")
elif project_info == []:
feed_utils.ok("project without dataset subtypes: %s" % project_info)
else:
feed_utils.ok("project dataset subtypes: %s" % project_info)
def test_project_list_all_users(project_id):
feed_utils.heading("Trying list users in project. project_id: %s" %
project_id)
project_info = sbi_project.get_users_in_project(project_id)
if project_info == None:
feed_utils.failed("failed getting project users")
elif project_info == []:
feed_utils.ok("project without users: %s" % project_info)
else:
feed_utils.ok("number of project users: %s" % len(project_info))
for user in project_info:
feed_utils.info("federated_id : %s" % user[u'federated_id'])
def test_quota_search(name):
feed_utils.heading("Trying quota search")
result = sbi_quotas.search_quotas(name)
if not result:
feed_utils.failed("search quotas failed")
else:
feed_utils.ok("found %s quotas from the search" % result[u'count'])
for quota in result[u'data']:
feed_utils.info("id: %d, quota_id: %d" %
(int(quota[u'id']), int(quota[u'quota_id'])))
if result[u'count'] == 1:
return int(quota[u'id']), int(quota[u'quota_id'])
return None
action="store_true",
help='turn verbosity on',
default=False)
parser.add_option('-d',
'--dryrun',
dest="dryrun",
action="store_true",
help='Dry run, don'
't affect persistence',
default=False)
# get options and arguments
(options, args) = parser.parse_args()
config.DRY_RUN = options.dryrun
feed_utils.VERBOSE = options.verbose
config.init()
feed_utils.heading("running user disk usage statistics")
total_size = 0
for nels_id in users.get_nels_ids():
try:
size = disk_facade.get_folder_size(
storage_facade.get_user_home(nels_id))
total_size = total_size + size
feed_utils.info("%s:%s" % (nels_id, size))
if not config.DRY_RUN:
stats.add_stat(stats.PERSONAL_DISK_USAGE, nels_id, size)
except Exception as ex:
feed_utils.error("error processing nels id: %s" % nels_id)
if not config.DRY_RUN:
stats.add_stat(stats.PERSONAL_DISK_USAGE_SUMMARY, -1, total_size)
parser.add_option('-v',
'--verbose',
dest="verbose",
action="store_true",
help='turn verbosity on',
default=False)
# get options and arguments
(options, args) = parser.parse_args()
feed_utils.VERBOSE = options.verbose
args_utils.require_args_length(parser, args, 1)
args_utils.require_arg_number(parser, args, 0, "project_id")
pid = args[0]
config.init()
feed_utils.heading("project info")
if not project_facade.project_exists(pid):
run_utils.exit_fail("project not found")
feed_utils.ok("project found\nname:\t%s\nhome folder:\t%s" %
(project_facade.project_name_by_id(pid),
project_facade.project_home(pid)))
feed_utils.push_in()
for role in [
project_facade.MembershipRoles.ADMIN,
project_facade.MembershipRoles.POWERUSER,
project_facade.MembershipRoles.MEMBER
]:
feed_utils.push_in()
'--notify',
dest="notify",
action="store_true",
help='turn notification on',
default=False)
(options, args) = parser.parse_args()
feed_utils.VERBOSE = options.verbose
feed_utils.NOTIFY = options.notify
args_utils.require_args_length(parser, args, 1)
email = str(args[0])
if not mail_utils.is_valid_email(email):
run_utils.exit_fail("invalid e-mail")
# list users
users_list_ok = test_users_list.test_users_list()
feed_utils.heading("A full life-cyle of user")
nels_id = users.register_user(users.NELS_IDP, email, "Test-added-user",
email, users.NORMAL_USER, True, "TEST")
if not nels_id:
run_utils.exit_fail("user registration failed")
test_users_list.test_user_display(nels_id)
ssh_ok = test_ssh(nels_id)
if ssh_ok:
test_copy(nels_id)
test_move(nels_id)
if not users.delete_user(nels_id):
run_utils.exit_fail("uanble to delete user")
feed_utils.ok("user deleted successfully")
def test_user_del(nels_id):
feed_utils.heading("Trying user deletion. nels_id : %s" %nels_id)
if not users.delete_user(nels_id):
feed_utils.failed("deletion of user failed")
else:
feed_utils.ok("deletion of user")
from optparse import OptionParser
import config
from facades import user_facade
from utils import feed_utils, args_utils
if __name__ == "__main__":
usage = 'usage: %prog [options] nels_id'
version = '%prog 1.0'
parser = OptionParser(usage=usage, version=version)
parser.add_option('-v',
'--verbose',
dest="verbose",
action="store_true",
help='turn verbosity on',
default=False)
# get options and arguments
(options, args) = parser.parse_args()
feed_utils.VERBOSE = options.verbose
args_utils.require_args_length(parser, args, 1)
args_utils.require_arg_number(parser, args, 0, "nels_id")
nels_id = args[0]
config.init()
feed_utils.heading("nels id to user name")
feed_utils.ok("nels id: %s : username: %s " %
(nels_id, user_facade.nels_id_to_username(nels_id)))
dest="verbose",
action="store_true",
help='turn verbosity on',
default=False)
parser.add_option('-d',
'--dryrun',
dest="dryrun",
action="store_true",
help='Dry run, don'
't affect persistence',
default=False)
# get options and arguments
(options, args) = parser.parse_args()
config.DRY_RUN = options.dryrun
feed_utils.VERBOSE = options.verbose
config.init()
feed_utils.heading("running project disk usage statistics")
total_size = 0
for project_id in projects.get_project_ids():
try:
size = disk_facade.get_folder_size(
storage_facade.get_project_home(project_id))
total_size = total_size + size
feed_utils.info("%s:%s" % (project_id, size))
if not config.DRY_RUN:
stats.add_stat(stats.PROJECT_DISK_USAGE, project_id, size)
except Exception as ex:
feed_utils.error("error processing project id: %s" % project_id)
if not config.DRY_RUN:
stats.add_stat(stats.PROJECT_DISK_USAGE_SUMMARY, -1, total_size)
def test_project_del(project_id):
feed_utils.heading("Trying project deletion. project_id : %s" % project_id)
if not project.delete_project(project_id):
feed_utils.failed("deletion of project failed")
else:
feed_utils.ok("deletion of project")
from optparse import OptionParser
import config
from facades import user_facade
from utils import feed_utils, args_utils, run_utils
if __name__ == "__main__":
usage = 'usage: %prog [options] nels_id'
version = '%prog 1.0'
parser = OptionParser(usage=usage, version=version)
parser.add_option('-v', '--verbose', dest="verbose", action="store_true", help='turn verbosity on', default=False)
# get options and arguments
(options, args) = parser.parse_args()
feed_utils.VERBOSE = options.verbose
args_utils.require_args_length(parser, args, 1)
args_utils.require_arg_number(parser, args, 0, "nels_id")
nels_id = args[0]
config.init()
feed_utils.heading("user info")
if not user_facade.user_exists(nels_id):
run_utils.exit_fail("user not found")
feed_utils.ok("user found\nnels-id:\t%s\nusername:\t%s\nhome folder:\t%s" % (
nels_id, user_facade.nels_id_to_username(nels_id), user_facade.user_home(nels_id)))
