def add_dataset_to_project(project_id, federated_id): try: json_dataset_type_array = get_dataset_types() json_body = { "data_set_type_id": json_dataset_type_array["data"][0]["id"], "name": "dataset integration", "description": "test" } headers = {'federated-id': federated_id} response = requests.post(config.sbi_url("projects/%s/datasets" % (project_id)), headers=headers, json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.created: return True else: feed_utils.info(response.status_code) except: feed_utils.error(sys.exc_info()[0]) return None
def get_subtypes_in_dataset_in_project(project_id, dataset_id): try: ids = [] response = requests.get(config.sbi_url( "projects/%s/datasets/%s/subtypes" % (project_id, dataset_id)), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: json_response = response.json() if json_response == []: """project dataset without subtypes""" return json_response else: for uid in json_response: if u'id' in uid: feed_utils.info(uid) ids.append(uid[u'id']) return ids else: feed_utils.debug(response.status_code) feed_utils.info(response.text) except: feed_utils.error(sys.exc_info()[0]) return None
def search_users(nels_id=None, idp_number=None, federated_id=None, name=None, email=None, user_type_id=None, is_active=None, affiliation=None): print email if True: json_body = {} json_body[u'federated_id'] = [] print json_body response = requests.post(config.sbi_url("/users/query"), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) print response if response.status_code == requests.codes.ok: print response.json() return response.json() else: feed_utils.error(sys.exc_info()[0]) return None
def test_ssh(nels_id): feed_utils.heading("ssh use case. nels_id: %s" % nels_id) credentail = ssh_utils.get_ssh_credentials(nels_id) if not credentail: feed_utils.failed("fetching key") return False feed_utils.ok("ssh key fetched") (host, username, key_file) = credentail (status, items, error) = run_utils.launch_remote_with_key(key_file, username, host, "ls ") if status != 0: feed_utils.error(error) return False if "Personal" in items: feed_utils.ok("Personal folder found") else: feed_utils.failed("Personal folder not found") if "Projects" in items: feed_utils.ok("Projects folder found") else: feed_utils.failed("Projects folder not found") feed_utils.info("cleaning key file") run_utils.launch_cmd("rm -f %s" % key_file) return True
def search_users(nels_id=None, idp_number=None, federated_id=None, name=None, email=None, user_type_id=None, is_active=None, affiliation=None): try: json_body = {} if nels_id: json_body['id'] = nels_id if idp_number: json_body['idpnumber'] = idp_number if federated_id: json_body['idpusername'] = federated_id if name: json_body['name'] = name if email: json_body['email'] = email if user_type_id: json_body['usertype'] = user_type_id if is_active: json_body['isactivie'] = is_active if affiliation: json_body['affiliation'] = affiliation response = requests.post(config.api_url("users/query"), json=json_body, auth=(config.API_KEY, config.API_SECRET)) if response.status_code == requests.codes.ok: return response.json() except: feed_utils.error(sys.exc_info()[0]) return None
def delete_quota(id): try: response = requests.delete(config.sbi_url("quotas/%s" % id), auth=(config.SBI_KEY, config.SBI_SECRET)) return response.status_code == requests.codes.no_content except: feed_utils.error(sys.exc_info()[0]) return False
def delete_project(project_id): try: response = requests.delete(config.api_url("projects/%s" % project_id), auth=(config.API_KEY, config.API_SECRET)) return response.status_code == requests.codes.no_content except: feed_utils.error(sys.exc_info()[0]) return False
def get_dataset_types(): try: response = requests.get(config.sbi_url("datasettypes"), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: return response.json() except: feed_utils.error(sys.exc_info()[0]) return None
def test_move(nels_id): feed_utils.heading("move use case. nels_id: %s" % nels_id) credentail = ssh_utils.get_ssh_credentials(nels_id) if not credentail: feed_utils.failed("fetching key") return False feed_utils.ok("ssh key fetched") (host, username, key_file) = credentail src_file = "Personal/dummy.txt" dest_dir = "Personal/dummy" if not ssh_utils.write_dummy_file(key_file, username, host, src_file, "5M"): feed_utils.failed("write test file") return False feed_utils.ok("write test file") if not ssh_utils.create_folder(key_file, username, host, dest_dir): feed_utils.failed("create test directory") return False feed_utils.ok("create test directory") job_id = job.add_job(job.STORAGE_MOVE, nels_id, [src_file], dest_dir) if job_id == -1: feed_utils.failed("job submission") return False feed_utils.ok("job submission. job-id: %s" % job_id) job.wait_for_job(job_id) [exit_code, output, error] = run_utils.launch_remote_with_key(key_file, username, host, "ls %s" % dest_dir) if exit_code != 0: feed_utils.error(error) return False if "dummy.txt" not in output: feed_utils.failed("verify moved file") return False feed_utils.ok("verify moved file") #validate file [exit_code, output, error] = run_utils.launch_remote_with_key(key_file, username, host, "rm -rf %s" % (dest_dir)) if not exit_code == 0: feed_utils.error(error) return False feed_utils.ok("cleanup") #clean up if not job.delete_job(job_id): feed_utils.failed("job delete") return False feed_utils.ok("job delete") return True
def get_quota(quota_id): try: response = requests.get(config.sbi_url("quotas/%s" % quota_id), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: return response.json() except: feed_utils.error(sys.exc_info()[0]) return None
def get_project(project_id): try: response = requests.get(config.api_url("projects/%s" % project_id), auth=(config.API_KEY, config.API_SECRET)) if response.status_code == requests.codes.ok: return response.json() except: feed_utils.error(sys.exc_info()[0]) return None
def add_user_to_project(project_id, user_id, membership_type): try: json_body = {"membership_type": membership_type} response = requests.post(config.api_url("projects/%s/users/%s" % (project_id, user_id)), json=json_body, auth=(config.API_KEY, config.API_SECRET)) return response.status_code == requests.codes.ok except: feed_utils.error(sys.exc_info()[0]) return None
def add_project(name, description): try: json_body = {"name": name, "description": description} response = requests.post(config.api_url("projects"), json=json_body, auth=(config.API_KEY, config.API_SECRET)) if response.status_code == requests.codes.created: return response.json()["id"] except: feed_utils.error(sys.exc_info()[0]) return None
def get_user_in_project(project_id, user_id): try: response = requests.get(config.api_url("projects/%s/users/%s" % (project_id, user_id)), auth=(config.API_KEY, config.API_SECRET)) if response.status_code == requests.codes.ok: json_response = response.json() return int(json_response["membership_type"]) except: feed_utils.error(sys.exc_info()[0]) return None
def test_copy_nels_to_sbi(nels_id): feed_utils.heading("copy use case nels to sbi. nels_id: %s" % nels_id) credentail = ssh_utils.get_ssh_credentials(nels_id) if not credentail: feed_utils.failed("fetching key") return False feed_utils.ok("ssh key fetched") (host, username, key_file) = credentail test_file = "dummy5.txt" src_dir = "Personal" src_file = "%s/%s" % (src_dir, test_file) dest_dir = '' if not ssh_utils.write_dummy_file(key_file, username, host, src_file, "5k"): # 5M feed_utils.failed("write test file") return False feed_utils.ok("write test file") dataset_id = "c16ff090-605d-4228-a8c9-3713a9a40e45" dataset_name = "regtest1feb" subtype = "Intensities" subtype_id = 1124874 job_id = job.add_sbi_job(job.SBI_PULL, nels_id, [test_file], [], host, username, file_utils.read_file_content(key_file), dataset_id, dataset_name, subtype, subtype_id, src_dir, "") if job_id == -1: feed_utils.failed("job submission") return False feed_utils.ok("job submission. job-id: %s" % job_id) job.wait_for_job(job_id) # validate file (to do) # clean up if not job.delete_job(job_id): feed_utils.failed("job delete") return False feed_utils.ok("job delete") # clean up on NeLS side [exit_code, output, error] = run_utils.launch_remote_with_key(key_file, username, host, "rm -rf %s " % (src_file)) if exit_code != 0: feed_utils.error(error) return False feed_utils.ok("clean up") return True
def get_subtype_in_dataset_in_project(project_id, dataset_id, subtype_id): try: response = requests.get( config.sbi_url("projects/%s/datasets/%s/subtypes/%s" % (project_id, dataset_id, subtype_id)), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: return response.json() except: feed_utils.error(sys.exc_info()[0]) return None
def delete_dataset_in_project(project_id, dataset_id): try: response = requests.delete(config.sbi_url("projects/%s/datasets/%s" % (project_id, dataset_id)), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.no_content: return True else: feed_utils.info(response.status_code) feed_utils.info(response.text) except: feed_utils.error(sys.exc_info()[0]) return None
def update_project(json_body_update_project, project_id): try: json_body = json_body_update_project response = requests.put(config.sbi_url("projects/%s" % project_id), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.no_content: return project_id except: feed_utils.error(sys.exc_info()[0]) return None
def get_project_ids(): try: ids = [] response = requests.get(config.api_url("projects/ids"), auth=(config.API_KEY, config.API_SECRET)) if response.status_code == requests.codes.ok: json_response = response.json() for uid in json_response: ids.append(uid[u'id']) return ids except: feed_utils.error(sys.exc_info()[0]) return None
def search_quotas(name): try: json_body = {'name': name, 'query': name} response = requests.post(config.sbi_url("quotas/query"), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: return response.json() feed_utils.debug(response.status_code) except: feed_utils.error(sys.exc_info()[0]) return None
def delete_user_from_project(project_id, federated_id): try: json_data_array = [federated_id] json_body = {"method": "delete", "data": json_data_array} response = requests.post(config.sbi_url("projects/%s/users/do" % (project_id)), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) return response.status_code == requests.codes.ok except: feed_utils.error(sys.exc_info()[0]) return None
def recompute_project_disc_usage(): try: json_body = {"method": "re_populate_project_disk_usage"} response = requests.post(config.sbi_url("projects/do"), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: return response.status_code == requests.codes.ok else: feed_utils.info(response.status_code) except: feed_utils.error(sys.exc_info()[0]) return False
def get_user_ids(): try: ids = [] response = requests.get(config.sbi_url("users"), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: json_response = response.json() for uid in json_response[u'data']: ids.append(uid[u'user_id']) return ids except: feed_utils.error(sys.exc_info()[0]) return None
def add_user_to_project(project_id, membership_type, federated_id): try: json_data_array = [{ "federated_id": federated_id, "role": membership_type }] json_body = {"method": "add", "data": json_data_array} response = requests.post(config.sbi_url("projects/%s/users/do" % (project_id)), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) return response.status_code == requests.codes.ok except: feed_utils.error(sys.exc_info()[0]) return None
def get_users_in_project(project_id): try: response = requests.get(config.sbi_url("projects/%s/users" % (project_id)), auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.ok: json_response = response.json() return json_response else: feed_utils.info(response.text) except: feed_utils.error(sys.exc_info()[0]) return None
def add_quota(name, federated_id): try: json_body = {u'name': u'%s' % name, u'description': u'added from integration test', u'federated_id': u'%s' % federated_id, u'quota_size': 1000000000000} response = requests.post(config.sbi_url("quotas/"), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.created: return requests.codes.created feed_utils.info(response.text) except: feed_utils.error(sys.exc_info()[0]) return None
def authenticate(self, request, environ, scopes, client): if request.method == "GET": if "nels_token" in request.query_arguments.keys(): feed_utils.info("nels_token %s" % request.get_param("nels_token")) try: expanded_token = json.loads( security_utils.decrypt( config.ENCRYPTION_KEY, request.get_param("nels_token"))) feed_utils.info(expanded_token) return (expanded_token, expanded_token["nels_id"]) except Exception as ex: feed_utils.error(ex.message) pass raise UserNotAuthenticated
def search_projects(project_id=None, name=None): try: json_body = {} if project_id: json_body['id'] = project_id if name: json_body['name'] = name response = requests.post(config.api_url("projects/query"), json=json_body, auth=(config.API_KEY, config.API_SECRET)) if response.status_code == requests.codes.ok: return response.json() except: feed_utils.error(sys.exc_info()[0]) return None
def delete_project(federated_id, project_id): try: json_body = {u'federated_id': u'%s' % federated_id} response = requests.post(config.sbi_url("projects/%s" % project_id), json=json_body, auth=(config.SBI_KEY, config.SBI_SECRET)) if response.status_code == requests.codes.no_content: return True else: feed_utils.debug(response.status_code) feed_utils.info(response.text) except: feed_utils.error(sys.exc_info()[0]) return None
def get_stats(context_id): try: stats = [] response = requests.get( nels_master_api.get_full_url("stats/contexts/%s" % context_id), auth=(nels_master_api.CLIENT_KEY, nels_master_api.CLIENT_SECRET)) if (response.status_code == requests.codes.ok): json_response = response.json() for lg in json_response: stats.append([ lg[u'id'], lg[u'statscontextid'], lg[u'targetid'], lg[u'value'], lg[u'statstime'] ]) return stats except Exception as ex: feed_utils.error(ex) return None