def update_policy(ranger_url, policy_id, policy_data, admin_username_password): """ Updating policy in ranger admin using policy id """ url = format("{ranger_url}/service/public/v2/api/policy/{policy_id}") base_64_string = base64.encodestring(admin_username_password).replace('\n', '') request = urllib2.Request(url, json.dumps(policy_data)) request.get_method = lambda: 'PUT' request.add_header('Content-Type', 'application/json') request.add_header('Accept', 'application/json') request.add_header('Authorization', format('Basic {base_64_string}')) try: result = openurl(request, timeout=20) response_code = result.getcode() if response_code == 200: Logger.info(format("Successfully updated policy in Ranger Admin")) return response_code else: Logger.error(format("Unable to update policy in Ranger Admin")) return None except urllib2.HTTPError as e: raise Fail("HTTPError while updating policy Reason = " + str(e.code)) except urllib2.URLError as e: raise Fail("URLError while updating policy. Reason = " + str(e.reason)) except TimeoutError: raise Fail("Connection timeout error while updating policy") except Exception as err: raise Fail(format("Error while updating policy. Reason = {err}"))
def get_user(ranger_url, user, admin_username_password): """ Get user from Ranger Admin """ url = format("{ranger_url}/service/xusers/users?name={user}") base_64_string = base64.encodestring(admin_username_password).replace('\n', '') request = urllib2.Request(url) request.add_header('Content-Type', 'application/json') request.add_header('Accept', 'application/json') request.add_header('Authorization', format('Basic {base_64_string}')) try: result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXUsers']) >= 0: for vxuser in response['vXUsers']: if vxuser['name'] == user: Logger.info(format("User with username {user} exists in Ranger Admin")) return vxuser Logger.info(format("User with username {user} doesn't exist in Ranger Admin")) return None else: Logger.error(format("Unable to get {user_name} user in Ranger Admin")) return None except urllib2.HTTPError as e: raise Fail("HTTPError while getting " + str(user) + " user. Reason = " + str(e.code)) except urllib2.URLError as e: raise Fail("URLError while getting " + str(user) + " user. Reason = " + str(e.reason)) except TimeoutError: raise Fail("Connection timeout error while getting " + str(user) + " user.") except Exception as err: raise Fail(format("Error while getting {user} user. Reason = {err}"))
def create_user(ranger_url, user_name, user_password, user_role, admin_username_password): """ Create user in Ranger Admin """ Logger.info(format("Creating {user_name} user with role as {user_role} in Ranger Admin")) url = format("{ranger_url}/service/xusers/secure/users") user = { 'name': user_name, 'password': user_password, 'firstName': user_name, 'userRoleList': [user_role], 'status': '1' } base_64_string = base64.encodestring(admin_username_password).replace('\n', '') request = urllib2.Request(url, json.dumps(user)) request.add_header('Content-Type', 'application/json') request.add_header('Accept', 'application/json') request.add_header('Authorization', format('Basic {base_64_string}')) try: result = openurl(request, timeout = 20) response_code = result.getcode() if response_code == 200: Logger.info(format("Successfully created {user_name} user with role {user_role} in Ranger Admin")) return response_code else: Logger.error(format("Unable to create {user_name} user in Ranger Admin")) return None except urllib2.HTTPError, e: raise Fail("HTTPError while creating " + str(user_name) + " user. Reason = " + str(e.code))
def get_repository_by_name_urllib2(self, name, component, status, usernamepassword): """ :param name: name of the component, from which, function will search in list of repositories :param component: component for which repository has to be checked :param status: active or inactive :param usernamepassword: user credentials using which repository needs to be searched :return Returns Ranger repository dict if found otherwise None """ try: searchRepoURL = self.urlReposPub + "?name=" + name + "&type=" + component + "&status=" + status request = urllib2.Request(searchRepoURL) base64string = base64.encodestring(usernamepassword).replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base64string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXRepositories']) > 0: for repo in response['vXRepositories']: repoDump = json.loads(json.JSONEncoder().encode(repo)) if repoDump['name'].lower() == name.lower(): return repoDump return None else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error getting {0} repository for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read())) else: raise Fail("Error getting {0} repository for component {1}. Reason - {2}.".format(name, component, e.reason))
def update_user_role(ranger_url, user_name, user_role, admin_username_password): """ Updating user role by username in Ranger Admin """ url = format("{ranger_url}/service/xusers/secure/users/roles/userName/{user_name}") role = { "vXStrings": [{"value": user_role}] } base_64_string = base64.encodestring(admin_username_password).replace('\n', '') request = urllib2.Request(url, json.dumps(role)) request.get_method = lambda: 'PUT' request.add_header('Content-Type', 'application/json') request.add_header('Accept', 'application/json') request.add_header('Authorization', format('Basic {base_64_string}')) try: result = openurl(request, timeout = 20) response_code = result.getcode() if response_code == 200: Logger.info(format("Successfully updated {user_name} user with role {user_role} in Ranger Admin")) return response_code else: Logger.error(format("Unable to update {user_name} user role with {user_role} in Ranger Admin")) return None except urllib2.HTTPError, e: raise Fail("HTTPError while updating " + str(user_name) + " user role to " + str(user_role) + ". Reason = " + str(e.code))
def get_policy_by_repo_name(self, name, component, status, usernamepassword): """ :param name: repository name :param component: component name for which policy needs to be searched :param status: true or false :param usernamepassword: user credentials using which policy needs to be searched :return Returns successful response else None """ try: searchPolicyURL = self.urlPolicies + "?repositoryName=" + name + "&repositoryType=" + component + "&isEnabled=" + status request = urllib2.Request(searchPolicyURL) base64string = base64.encodestring(usernamepassword).replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base64string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXPolicies']) > 0: return response['vXPolicies'] else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error getting policy from repository {0} for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read())) else: raise Fail("Error getting policy from repository {0} for component {1}. Reason - {2}.".format(name, component, e.reason))
def create_repository_urllib2(self, data, usernamepassword): """ :param data: json object to create repository :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns created Ranger repository object """ try: search_repo_url = self.url_repos_pub base_64_string = base64.encodestring( '{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(search_repo_url, data, headers) request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: Logger.info('Repository created Successfully') return response else: raise Fail('Repository creation failed') except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail( "Error creating repository. Http status code - {0}. \n {1}" .format(e.code, e.read())) else: raise Fail("Error creating repository. Reason - {0}.".format( e.reason))
def get_repository_by_name_urllib2(self, name, component, status, usernamepassword): """ :param name: name of the component, from which, function will search in list of repositories :param component:, component for which repository has to be checked :param status: active or inactive :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns Ranger repository object if found otherwise None """ try: search_repo_url = self.url_repos_pub + "?name=" + name + "&type=" + component + "&status=" + status request = urllib2.Request(search_repo_url) base_64_string = base64.encodestring(usernamepassword).replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response) > 0: for repo in response: repo_dump = json.loads(json.JSONEncoder().encode(repo)) if repo_dump['name'].lower() == name.lower(): return repo_dump return None else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error getting {0} repository for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read())) else: raise Fail("Error getting {0} repository for component {1}. Reason - {2}.".format(name, component, e.reason))
def create_repository_urllib2(self, data, usernamepassword): """ :param data: json object to create repository :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns created Ranger repository object """ try: search_repo_url = self.url_repos_pub base_64_string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(search_repo_url, data, headers) request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: Logger.info('Repository created Successfully') return response else: raise Fail('Repository creation failed') except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error creating repository. Reason - {0}.".format(e.reason))
def get_ranger_service_default_policy(ranger_url, service_name, admin_username_password, resource_list = ['database', 'table', 'column']): """ Get Policies by service name """ url = format("{ranger_url}/service/public/v2/api/service/{service_name}/policy") base_64_string = base64.encodestring(admin_username_password).replace('\n', '') request = urllib2.Request(url) request.add_header('Content-Type', 'application/json') request.add_header('Accept', 'application/json') request.add_header('Authorization', format('Basic {base_64_string}')) try: result = openurl(request, timeout = 20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response) > 0: for policy in response: count = 0 for resource in resource_list: if resource in policy['resources'] and '*' in policy['resources'][resource]['values']: count = count + 1 if count == len(resource_list): Logger.info(format("Default policy exists in {service_name} in Ranger Admin for resources {resource_list}")) return policy Logger.info(format("Default policy doesn't exists in {service_name} in Ranger Admin for resources {resource_list}")) return False else: Logger.error(format("Unable to get default policy from {service_name} service.")) return None except urllib2.HTTPError, e: raise Fail("HTTPError while getting default policy from " + str(service_name) + " service. Reason = " + str(e.code))
def update_ranger_policy(self, policyId, data, usernamepassword): """ :param policyId: policy id which needs to be updated :param data: policy data that needs to be updated :param usernamepassword: user credentials using which policy needs to be updated :return Returns successful response and response code else None """ try: searchRepoURL = self.urlPolicies + "/" + str(policyId) base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(searchRepoURL, data, headers) request.add_header("Authorization", "Basic {0}".format(base64string)) request.get_method = lambda: 'PUT' result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: Logger.info('Policy updated Successfully') return response_code else: Logger.error('Update Policy failed') return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error updating policy. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error updating policy. Reason - {0}.".format(e.reason))
def check_nifi_portal_with_python(url): try: request = urllib2.Request(url) result = openurl(request, timeout=20) response_code = result.getcode() if response_code == 200 or response_code == 401: Logger.info("Nifi portal {0} is up. Response code {1}".format( url, response_code)) else: raise Fail("Error connecting to {0}. Response code {1}".format( url, response_code)) except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): if e.code == 401: Logger.info( "Nifi portal {0} is up. Response code {1}".format( url, e.code)) else: raise Fail( "Error connecting to {0}. Http status code - {1}. \n {2}" .format(url, e.code, e.read())) elif e.reason and "violation of protocol" in str(e.reason): Logger.info("Ignore certificate validation error - {0}".format( e.reason)) pass else: raise Fail("Error connecting to {0}. Reason - {1}.".format( url, e.reason))
def create_repository_urllib2(self, data, usernamepassword, policy_user): """ :param data: repository dict :param usernamepassword: user credentials using which repository needs to be created :param policy_user: use this policy user for policies that will be used during repository creation :return Returns created repository response else None """ try: searchRepoURL = self.urlReposPub base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(searchRepoURL, data, headers) request.add_header("Authorization", "Basic {0}".format(base64string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: Logger.info('Repository created Successfully') # Get Policies repoData = json.loads(data) repoName = repoData['name'] typeOfPolicy = repoData['repositoryType'] # Get Policies by repo name policyList = self.get_policy_by_repo_name(name=repoName, component=typeOfPolicy, status="true", usernamepassword=usernamepassword) if policyList is not None and (len(policyList)) > 0: policiesUpdateCount = 0 for policy in policyList: updatedPolicyObj = self.get_policy_params(typeOfPolicy, policy, policy_user) policyResCode = self.update_ranger_policy(updatedPolicyObj['id'], json.dumps(updatedPolicyObj), usernamepassword) if policyResCode == 200: policiesUpdateCount = policiesUpdateCount + 1 else: Logger.info('Policy Update failed') # Check for count of updated policies if len(policyList) == policiesUpdateCount: Logger.info( "Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions") return response else: return None else: Logger.info("Policies not found for the newly created Repository") return None else: Logger.info('Repository creation failed') return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error creating repository. Reason - {0}.".format(e.reason))
def update_repository_urllib2(self, component, repo_name, repo_properties, admin_user, admin_password, force_rename=False): """ param component: name of service supported by Ranger Admin param repo_name: name of service name that needs to be updated param repo_properties: configs that needs to be updated for given service name param admin_user: user having role admin in Ranger Admin param admin_password: password of the admin user used param force_rename: flag to forcefully rename service name if required during updation """ try: update_repo_url = self.url_repos_pub + "/name/" + repo_name if force_rename: update_repo_url = update_repo_url + "?forceRename=true" repo_update_data = json.dumps(repo_properties) usernamepassword = admin_user + ":" + admin_password base_64_string = base64.encodestring( "{0}".format(usernamepassword)).replace("\n", "") headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' } request = urllib2.Request(update_repo_url, repo_update_data, headers) request.add_header("Authorization", "Basic {0}".format(base_64_string)) request.get_method = lambda: 'PUT' result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: Logger.info( "Service name {0} updated successfully on Ranger Admin for service {1}" .format(repo_name, component)) return response else: raise Fail( "Service name {0} updation failed on Ranger Admin for service {1}" .format(repo_name, component)) except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail( "Error updating service name {0} on Ranger Admin for service {1}. Http status code - {2} \n {3}" .format(repo_name, component, e.code, e.read())) else: raise Fail( "Error updating service name {0} on Ranger Admin for service {1}. Reason - {2}" .format(repo_name, component, e.reason))
def check_ranger_login_urllib2(self, url): """ :param url: ranger admin host url :return Returns login check response """ try: response = openurl(url, timeout=20) response_code = response.getcode() return response_code except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Connection to Ranger Admin failed. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Connection to Ranger Admin failed. Reason - {0}.".format(e.reason))
def check_ranger_login_urllib2(self, url): """ :param url: ranger admin host url :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns login check response """ try: response = openurl(url, timeout=20) response_code = response.getcode() return response_code except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Connection failed to Ranger Admin. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Connection failed to Ranger Admin. Reason - {0}.".format(e.reason))
def create_ambari_admin_user(self,ambari_admin_username, ambari_admin_password,usernamepassword): """ :param ambari_admin_username: username of user to be created :param ambari_admin_username: user password of user to be created :return Returns response code for successful user creation else None """ flag_ambari_admin_present = False match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password) if match is None: raise Fail('Invalid password given for Ranger Admin user for Ambari') try: url = self.urlUsers + '?name=' + str(ambari_admin_username) request = urllib2.Request(url) base64string = base64.encodestring(usernamepassword).replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base64string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXUsers']) >= 0: for vxuser in response['vXUsers']: if vxuser['name'] == ambari_admin_username: flag_ambari_admin_present = True break else: flag_ambari_admin_present = False if flag_ambari_admin_present: Logger.info(ambari_admin_username + ' user already exists.') return response_code else: Logger.info(ambari_admin_username + ' user is not present, creating user using given configurations') url = self.urlSecUsers admin_user = dict() admin_user['status'] = 1 admin_user['userRoleList'] = ['ROLE_SYS_ADMIN'] admin_user['name'] = ambari_admin_username admin_user['password'] = ambari_admin_password admin_user['description'] = ambari_admin_username admin_user['firstName'] = ambari_admin_username data = json.dumps(admin_user) base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(url, data, headers) request.add_header("Authorization", "Basic {0}".format(base64string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200 and response is not None: Logger.info('Ambari admin user creation successful.') return response_code else: Logger.info('Ambari admin user creation failed.') return None else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason))
def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, usernamepassword): """ :param ambari_admin_username: username of user to be created :param ambari_admin_password: user password of user to be created :param usernamepassword: user credentials using which repository needs to be searched. :return: Returns user credentials if user exist otherwise rerutns credentials of created user. """ flag_ambari_admin_present = False match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password) if match is None: raise Fail('Invalid password given for Ranger Admin user for Ambari') try: url = self.url_users + '?name=' + str(ambari_admin_username) request = urllib2.Request(url) base_64_string = base64.encodestring(usernamepassword).replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) if response_code == 200 and len(response['vXUsers']) >= 0: for vxuser in response['vXUsers']: if vxuser['name'] == ambari_admin_username: flag_ambari_admin_present = True break else: flag_ambari_admin_present = False if flag_ambari_admin_present: Logger.info(ambari_admin_username + ' user already exists.') return response_code else: Logger.info(ambari_admin_username + ' user is not present, creating user using given configurations') url = self.url_sec_users admin_user = dict() admin_user['status'] = 1 admin_user['userRoleList'] = ['ROLE_SYS_ADMIN'] admin_user['name'] = ambari_admin_username admin_user['password'] = ambari_admin_password admin_user['description'] = ambari_admin_username admin_user['firstName'] = ambari_admin_username data = json.dumps(admin_user) base_64_string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib2.Request(url, data, headers) request.add_header("Authorization", "Basic {0}".format(base_64_string)) result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200 and response is not None: Logger.info('Ambari admin user creation successful.') return response_code else: Logger.error('Ambari admin user creation failed.') return None else: return None except urllib2.URLError, e: if isinstance(e, urllib2.HTTPError): raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read())) else: raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason))