def add_dataset_to_project(project_id, federated_id):
try:
json_dataset_type_array = get_dataset_types()
json_body = {
"data_set_type_id": json_dataset_type_array["data"][0]["id"],
"name": "dataset integration",
"description": "test"
}
headers = {'federated-id': federated_id}
response = requests.post(config.sbi_url("projects/%s/datasets" %
(project_id)),
headers=headers,
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.created:
return True
else:
feed_utils.info(response.status_code)
except:
feed_utils.error(sys.exc_info()[0])
return None
def get_subtypes_in_dataset_in_project(project_id, dataset_id):
try:
ids = []
response = requests.get(config.sbi_url(
"projects/%s/datasets/%s/subtypes" % (project_id, dataset_id)),
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
json_response = response.json()
if json_response == []:
"""project dataset without subtypes"""
return json_response
else:
for uid in json_response:
if u'id' in uid:
feed_utils.info(uid)
ids.append(uid[u'id'])
return ids
else:
feed_utils.debug(response.status_code)
feed_utils.info(response.text)
except:
feed_utils.error(sys.exc_info()[0])
return None
def search_users(nels_id=None,
idp_number=None,
federated_id=None,
name=None,
email=None,
user_type_id=None,
is_active=None,
affiliation=None):
print email
if True:
json_body = {}
json_body[u'federated_id'] = []
print json_body
response = requests.post(config.sbi_url("/users/query"),
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
print response
if response.status_code == requests.codes.ok:
print response.json()
return response.json()
else:
feed_utils.error(sys.exc_info()[0])
return None
def test_ssh(nels_id):
feed_utils.heading("ssh use case. nels_id: %s" % nels_id)
credentail = ssh_utils.get_ssh_credentials(nels_id)
if not credentail:
feed_utils.failed("fetching key")
return False
feed_utils.ok("ssh key fetched")
(host, username, key_file) = credentail
(status, items,
error) = run_utils.launch_remote_with_key(key_file, username, host, "ls ")
if status != 0:
feed_utils.error(error)
return False
if "Personal" in items:
feed_utils.ok("Personal folder found")
else:
feed_utils.failed("Personal folder not found")
if "Projects" in items:
feed_utils.ok("Projects folder found")
else:
feed_utils.failed("Projects folder not found")
feed_utils.info("cleaning key file")
run_utils.launch_cmd("rm -f %s" % key_file)
return True
def search_users(nels_id=None,
idp_number=None,
federated_id=None,
name=None,
email=None,
user_type_id=None,
is_active=None,
affiliation=None):
try:
json_body = {}
if nels_id:
json_body['id'] = nels_id
if idp_number:
json_body['idpnumber'] = idp_number
if federated_id:
json_body['idpusername'] = federated_id
if name:
json_body['name'] = name
if email:
json_body['email'] = email
if user_type_id:
json_body['usertype'] = user_type_id
if is_active:
json_body['isactivie'] = is_active
if affiliation:
json_body['affiliation'] = affiliation
response = requests.post(config.api_url("users/query"),
json=json_body,
auth=(config.API_KEY, config.API_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
except:
feed_utils.error(sys.exc_info()[0])
return None
def delete_quota(id):
try:
response = requests.delete(config.sbi_url("quotas/%s" % id), auth=(config.SBI_KEY, config.SBI_SECRET))
return response.status_code == requests.codes.no_content
except:
feed_utils.error(sys.exc_info()[0])
return False
def delete_project(project_id):
try:
response = requests.delete(config.api_url("projects/%s" % project_id),
auth=(config.API_KEY, config.API_SECRET))
return response.status_code == requests.codes.no_content
except:
feed_utils.error(sys.exc_info()[0])
return False
def get_dataset_types():
try:
response = requests.get(config.sbi_url("datasettypes"),
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
except:
feed_utils.error(sys.exc_info()[0])
return None
def test_move(nels_id):
feed_utils.heading("move use case. nels_id: %s" % nels_id)
credentail = ssh_utils.get_ssh_credentials(nels_id)
if not credentail:
feed_utils.failed("fetching key")
return False
feed_utils.ok("ssh key fetched")
(host, username, key_file) = credentail
src_file = "Personal/dummy.txt"
dest_dir = "Personal/dummy"
if not ssh_utils.write_dummy_file(key_file, username, host, src_file,
"5M"):
feed_utils.failed("write test file")
return False
feed_utils.ok("write test file")
if not ssh_utils.create_folder(key_file, username, host, dest_dir):
feed_utils.failed("create test directory")
return False
feed_utils.ok("create test directory")
job_id = job.add_job(job.STORAGE_MOVE, nels_id, [src_file], dest_dir)
if job_id == -1:
feed_utils.failed("job submission")
return False
feed_utils.ok("job submission. job-id: %s" % job_id)
job.wait_for_job(job_id)
[exit_code, output,
error] = run_utils.launch_remote_with_key(key_file, username, host,
"ls %s" % dest_dir)
if exit_code != 0:
feed_utils.error(error)
return False
if "dummy.txt" not in output:
feed_utils.failed("verify moved file")
return False
feed_utils.ok("verify moved file")
#validate file
[exit_code, output,
error] = run_utils.launch_remote_with_key(key_file, username, host,
"rm -rf %s" % (dest_dir))
if not exit_code == 0:
feed_utils.error(error)
return False
feed_utils.ok("cleanup")
#clean up
if not job.delete_job(job_id):
feed_utils.failed("job delete")
return False
feed_utils.ok("job delete")
return True
def get_quota(quota_id):
try:
response = requests.get(config.sbi_url("quotas/%s" % quota_id), auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
except:
feed_utils.error(sys.exc_info()[0])
return None
def get_project(project_id):
try:
response = requests.get(config.api_url("projects/%s" % project_id),
auth=(config.API_KEY, config.API_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
except:
feed_utils.error(sys.exc_info()[0])
return None
def add_user_to_project(project_id, user_id, membership_type):
try:
json_body = {"membership_type": membership_type}
response = requests.post(config.api_url("projects/%s/users/%s" %
(project_id, user_id)),
json=json_body,
auth=(config.API_KEY, config.API_SECRET))
return response.status_code == requests.codes.ok
except:
feed_utils.error(sys.exc_info()[0])
return None
def add_project(name, description):
try:
json_body = {"name": name, "description": description}
response = requests.post(config.api_url("projects"),
json=json_body,
auth=(config.API_KEY, config.API_SECRET))
if response.status_code == requests.codes.created:
return response.json()["id"]
except:
feed_utils.error(sys.exc_info()[0])
return None
def get_user_in_project(project_id, user_id):
try:
response = requests.get(config.api_url("projects/%s/users/%s" %
(project_id, user_id)),
auth=(config.API_KEY, config.API_SECRET))
if response.status_code == requests.codes.ok:
json_response = response.json()
return int(json_response["membership_type"])
except:
feed_utils.error(sys.exc_info()[0])
return None
def test_copy_nels_to_sbi(nels_id):
feed_utils.heading("copy use case nels to sbi. nels_id: %s" % nels_id)
credentail = ssh_utils.get_ssh_credentials(nels_id)
if not credentail:
feed_utils.failed("fetching key")
return False
feed_utils.ok("ssh key fetched")
(host, username, key_file) = credentail
test_file = "dummy5.txt"
src_dir = "Personal"
src_file = "%s/%s" % (src_dir, test_file)
dest_dir = ''
if not ssh_utils.write_dummy_file(key_file, username, host, src_file,
"5k"): # 5M
feed_utils.failed("write test file")
return False
feed_utils.ok("write test file")
dataset_id = "c16ff090-605d-4228-a8c9-3713a9a40e45"
dataset_name = "regtest1feb"
subtype = "Intensities"
subtype_id = 1124874
job_id = job.add_sbi_job(job.SBI_PULL, nels_id, [test_file], [], host,
username, file_utils.read_file_content(key_file),
dataset_id, dataset_name, subtype, subtype_id,
src_dir, "")
if job_id == -1:
feed_utils.failed("job submission")
return False
feed_utils.ok("job submission. job-id: %s" % job_id)
job.wait_for_job(job_id)
# validate file (to do)
# clean up
if not job.delete_job(job_id):
feed_utils.failed("job delete")
return False
feed_utils.ok("job delete")
# clean up on NeLS side
[exit_code, output,
error] = run_utils.launch_remote_with_key(key_file, username, host,
"rm -rf %s " % (src_file))
if exit_code != 0:
feed_utils.error(error)
return False
feed_utils.ok("clean up")
return True
def get_subtype_in_dataset_in_project(project_id, dataset_id, subtype_id):
try:
response = requests.get(
config.sbi_url("projects/%s/datasets/%s/subtypes/%s" %
(project_id, dataset_id, subtype_id)),
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
except:
feed_utils.error(sys.exc_info()[0])
return None
def delete_dataset_in_project(project_id, dataset_id):
try:
response = requests.delete(config.sbi_url("projects/%s/datasets/%s" %
(project_id, dataset_id)),
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.no_content:
return True
else:
feed_utils.info(response.status_code)
feed_utils.info(response.text)
except:
feed_utils.error(sys.exc_info()[0])
return None
def update_project(json_body_update_project, project_id):
try:
json_body = json_body_update_project
response = requests.put(config.sbi_url("projects/%s" % project_id),
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.no_content:
return project_id
except:
feed_utils.error(sys.exc_info()[0])
return None
def get_project_ids():
try:
ids = []
response = requests.get(config.api_url("projects/ids"),
auth=(config.API_KEY, config.API_SECRET))
if response.status_code == requests.codes.ok:
json_response = response.json()
for uid in json_response:
ids.append(uid[u'id'])
return ids
except:
feed_utils.error(sys.exc_info()[0])
return None
def search_quotas(name):
try:
json_body = {'name': name, 'query': name}
response = requests.post(config.sbi_url("quotas/query"), json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
feed_utils.debug(response.status_code)
except:
feed_utils.error(sys.exc_info()[0])
return None
def delete_user_from_project(project_id, federated_id):
try:
json_data_array = [federated_id]
json_body = {"method": "delete", "data": json_data_array}
response = requests.post(config.sbi_url("projects/%s/users/do" %
(project_id)),
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
return response.status_code == requests.codes.ok
except:
feed_utils.error(sys.exc_info()[0])
return None
def recompute_project_disc_usage():
try:
json_body = {"method": "re_populate_project_disk_usage"}
response = requests.post(config.sbi_url("projects/do"),
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
return response.status_code == requests.codes.ok
else:
feed_utils.info(response.status_code)
except:
feed_utils.error(sys.exc_info()[0])
return False
def get_user_ids():
try:
ids = []
response = requests.get(config.sbi_url("users"),
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
json_response = response.json()
for uid in json_response[u'data']:
ids.append(uid[u'user_id'])
return ids
except:
feed_utils.error(sys.exc_info()[0])
return None
def add_user_to_project(project_id, membership_type, federated_id):
try:
json_data_array = [{
"federated_id": federated_id,
"role": membership_type
}]
json_body = {"method": "add", "data": json_data_array}
response = requests.post(config.sbi_url("projects/%s/users/do" %
(project_id)),
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
return response.status_code == requests.codes.ok
except:
feed_utils.error(sys.exc_info()[0])
return None
def get_users_in_project(project_id):
try:
response = requests.get(config.sbi_url("projects/%s/users" %
(project_id)),
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.ok:
json_response = response.json()
return json_response
else:
feed_utils.info(response.text)
except:
feed_utils.error(sys.exc_info()[0])
return None
def add_quota(name, federated_id):
try:
json_body = {u'name': u'%s' % name,
u'description': u'added from integration test',
u'federated_id': u'%s' % federated_id,
u'quota_size': 1000000000000}
response = requests.post(config.sbi_url("quotas/"), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.created:
return requests.codes.created
feed_utils.info(response.text)
except:
feed_utils.error(sys.exc_info()[0])
return None
def authenticate(self, request, environ, scopes, client):
if request.method == "GET":
if "nels_token" in request.query_arguments.keys():
feed_utils.info("nels_token %s" %
request.get_param("nels_token"))
try:
expanded_token = json.loads(
security_utils.decrypt(
config.ENCRYPTION_KEY,
request.get_param("nels_token")))
feed_utils.info(expanded_token)
return (expanded_token, expanded_token["nels_id"])
except Exception as ex:
feed_utils.error(ex.message)
pass
raise UserNotAuthenticated
def search_projects(project_id=None, name=None):
try:
json_body = {}
if project_id:
json_body['id'] = project_id
if name:
json_body['name'] = name
response = requests.post(config.api_url("projects/query"),
json=json_body,
auth=(config.API_KEY, config.API_SECRET))
if response.status_code == requests.codes.ok:
return response.json()
except:
feed_utils.error(sys.exc_info()[0])
return None
def delete_project(federated_id, project_id):
try:
json_body = {u'federated_id': u'%s' % federated_id}
response = requests.post(config.sbi_url("projects/%s" % project_id),
json=json_body,
auth=(config.SBI_KEY, config.SBI_SECRET))
if response.status_code == requests.codes.no_content:
return True
else:
feed_utils.debug(response.status_code)
feed_utils.info(response.text)
except:
feed_utils.error(sys.exc_info()[0])
return None
def get_stats(context_id):
try:
stats = []
response = requests.get(
nels_master_api.get_full_url("stats/contexts/%s" % context_id),
auth=(nels_master_api.CLIENT_KEY, nels_master_api.CLIENT_SECRET))
if (response.status_code == requests.codes.ok):
json_response = response.json()
for lg in json_response:
stats.append([
lg[u'id'], lg[u'statscontextid'], lg[u'targetid'],
lg[u'value'], lg[u'statstime']
])
return stats
except Exception as ex:
feed_utils.error(ex)
return None
