def get_repository_by_name_urllib2(self, name, component, status, usernamepassword):
"""
:param name: name of the component, from which, function will search in list of repositories
:param component: component for which repository has to be checked
:param status: active or inactive
:param usernamepassword: user credentials using which repository needs to be searched
:return Returns Ranger repository dict if found otherwise None
"""
try:
searchRepoURL = self.urlReposPub + "?name=" + name + "&type=" + component + "&status=" + status
request = urllib2.Request(searchRepoURL)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization", "Basic {0}".format(base64string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXRepositories']) > 0:
for repo in response['vXRepositories']:
repoDump = json.loads(json.JSONEncoder().encode(repo))
if repoDump['name'].lower() == name.lower():
return repoDump
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail("Error getting {0} repository for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read()))
else:
raise Fail("Error getting {0} repository for component {1}. Reason - {2}.".format(name, component, e.reason))
def create_repo(url, data, usernamepassword):
try:
base_url = url + '/service/public/v2/api/service'
base64string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(base_url, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
result = urllib2.urlopen(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Repository created Successfully')
return True
else:
Logger.info('Repository not created')
return False
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail(
"Error creating service. Http status code - {0}. \n {1}".
format(e.code, e.read()))
else:
raise Fail("Error creating service. Reason - {0}.".format(
e.reason))
def update_ranger_policy(self, policyId, data, usernamepassword):
"""
:param policyId: policy id which needs to be updated
:param data: policy data that needs to be updated
:param usernamepassword: user credentials using which policy needs to be updated
:return Returns successful response and response code else None
"""
try:
searchRepoURL = self.urlPolicies + "/" + str(policyId)
base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
request.get_method = lambda: 'PUT'
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Policy updated Successfully')
return response_code
else:
Logger.error('Update Policy failed')
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail("Error updating policy. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
raise Fail("Error updating policy. Reason - {0}.".format(e.reason))
def update_ranger_policy(self, policyId, data, usernamepassword):
try:
searchRepoURL = self.urlPolicies + "/" + str(policyId)
base64string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
request.add_header("Authorization",
"Basic {0}".format(base64string))
request.get_method = lambda: 'PUT'
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Policy updated Successfully')
return response_code, response
else:
Logger.error('Update Policy failed')
return None, None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("HTTP Code: {0}".format(e.code))
Logger.error("HTTP Data: {0}".format(e.read()))
else:
Logger.error("Error: {0}".format(e.reason))
return None, None
def get_repository_by_name_urllib2(self, name, component, status,
usernamepassword):
try:
searchRepoURL = self.urlReposPub + "?name=" + name + "&type=" + component + "&status=" + status
request = urllib2.Request(searchRepoURL)
base64string = base64.encodestring(usernamepassword).replace(
'\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization",
"Basic {0}".format(base64string))
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXRepositories']) > 0:
for repo in response['vXRepositories']:
repoDump = json.loads(json.JSONEncoder().encode(repo))
if repoDump['name'] == name:
return repoDump
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("HTTP Code: {0}".format(e.code))
Logger.error("HTTP Data: {0}".format(e.read()))
else:
Logger.error("Error : {0}".format(e.reason))
return None
def create_repository_urllib2(self, data, usernamepassword):
"""
:param data: json object to create repository
:param usernamepassword: user credentials using which repository needs to be searched.
:return: Returns created Ranger repository object
"""
try:
search_repo_url = self.url_repos_pub
base_64_string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(search_repo_url, data, headers)
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Repository created Successfully')
return response
else:
raise Fail('Repository creation failed')
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail(
"Error creating repository. Http status code - {0}. \n {1}"
.format(e.code, e.read()))
else:
raise Fail("Error creating repository. Reason - {0}.".format(
e.reason))
def get_repository_by_name_urllib2(self, name, component, status, usernamepassword):
"""
:param name: name of the component, from which, function will search in list of repositories
:param component:, component for which repository has to be checked
:param status: active or inactive
:param usernamepassword: user credentials using which repository needs to be searched.
:return: Returns Ranger repository object if found otherwise None
"""
try:
search_repo_url = self.url_repos_pub + "?name=" + name + "&type=" + component + "&status=" + status
request = urllib2.Request(search_repo_url)
base_64_string = base64.encodestring(usernamepassword).replace('\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization", "Basic {0}".format(base_64_string))
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response) > 0:
for repo in response:
repo_dump = json.loads(json.JSONEncoder().encode(repo))
if repo_dump['name'] == name:
return repo_dump
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("HTTP Code: {0}".format(e.code))
Logger.error("HTTP Data: {0}".format(e.read()))
else:
Logger.error("Error : {0}".format(e.reason))
return None
def create_repository_urllib2(self, data, usernamepassword, policy_user):
"""
param data: repository dict
param usernamepassword: user credentials using which repository needs to be created
param policy_user: use this policy user for policies that will be used during repository creation
return: Returns created repository response else None
"""
try:
searchRepoURL = self.urlReposPub
base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(searchRepoURL, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Repository created Successfully')
# Get Policies
repoData = json.loads(data)
repoName = repoData['name']
typeOfPolicy = repoData['repositoryType']
# Get Policies by repo name
policyList = self.get_policy_by_repo_name(name=repoName, component=typeOfPolicy, status="true",
usernamepassword=usernamepassword)
if policyList is not None and (len(policyList)) > 0:
policiesUpdateCount = 0
for policy in policyList:
updatedPolicyObj = self.get_policy_params(typeOfPolicy, policy, policy_user)
policyResCode, policyResponse = self.update_ranger_policy(updatedPolicyObj['id'],
json.dumps(updatedPolicyObj), usernamepassword)
if policyResCode == 200:
policiesUpdateCount = policiesUpdateCount + 1
else:
Logger.info('Policy Update failed')
# Check for count of updated policies
if len(policyList) == policiesUpdateCount:
Logger.info(
"Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions")
return response
else:
return None
else:
Logger.info("Policies not found for the newly created Repository")
return None
else:
Logger.info('Repository creation failed')
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
Logger.error("Error creating repository. Reason - {0}.".format(e.reason))
return None
def update_repository_urllib2(self,
component,
repo_name,
repo_properties,
admin_user,
admin_password,
force_rename=False):
"""
param component: name of service supported by Ranger Admin
param repo_name: name of service name that needs to be updated
param repo_properties: configs that needs to be updated for given service name
param admin_user: user having role admin in Ranger Admin
param admin_password: password of the admin user used
param force_rename: flag to forcefully rename service name if required during updation
"""
try:
update_repo_url = self.url_repos_pub + "/name/" + repo_name
if force_rename:
update_repo_url = update_repo_url + "?forceRename=true"
repo_update_data = json.dumps(repo_properties)
usernamepassword = admin_user + ":" + admin_password
base_64_string = base64.encodestring(
"{0}".format(usernamepassword)).replace("\n", "")
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
request = urllib2.Request(update_repo_url, repo_update_data,
headers)
request.add_header("Authorization",
"Basic {0}".format(base_64_string))
request.get_method = lambda: 'PUT'
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info(
"Service name {0} updated successfully on Ranger Admin for service {1}"
.format(repo_name, component))
return response
else:
raise Fail(
"Service name {0} updation failed on Ranger Admin for service {1}"
.format(repo_name, component))
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail(
"Error updating service name {0} on Ranger Admin for service {1}. Http status code - {2} \n {3}"
.format(repo_name, component, e.code, e.read()))
else:
raise Fail(
"Error updating service name {0} on Ranger Admin for service {1}. Reason - {2}"
.format(repo_name, component, e.reason))
def create_repo(url, data, usernamepassword):
try:
base_url = url + '/service/public/v2/api/service'
base64string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(base_url, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200:
Logger.info('Repository created Successfully')
else:
Logger.info('Repository not created')
except urllib2.URLError, e:
raise Fail('Repository creation failed, {0}'.format(str(e)))
def create_ambari_admin_user(self,ambari_admin_username, ambari_admin_password,usernamepassword):
"""
:param ambari_admin_username: username of user to be created
:param ambari_admin_username: user password of user to be created
:return Returns response code for successful user creation else None
"""
flag_ambari_admin_present = False
match = re.match('[a-zA-Z0-9_\S]+$', ambari_admin_password)
if match is None:
raise Fail('Invalid password given for Ranger Admin user for Ambari')
try:
url = self.urlUsers + '?name=' + str(ambari_admin_username)
request = urllib2.Request(url)
base64string = base64.encodestring(usernamepassword).replace('\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization", "Basic {0}".format(base64string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXUsers']) >= 0:
for vxuser in response['vXUsers']:
if vxuser['name'] == ambari_admin_username:
flag_ambari_admin_present = True
break
else:
flag_ambari_admin_present = False
if flag_ambari_admin_present:
Logger.info(ambari_admin_username + ' user already exists.')
return response_code
else:
Logger.info(ambari_admin_username + ' user is not present, creating user using given configurations')
url = self.urlSecUsers
admin_user = dict()
admin_user['status'] = 1
admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
admin_user['name'] = ambari_admin_username
admin_user['password'] = ambari_admin_password
admin_user['description'] = ambari_admin_username
admin_user['firstName'] = ambari_admin_username
data = json.dumps(admin_user)
base64string = base64.encodestring('{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(url, data, headers)
request.add_header("Authorization", "Basic {0}".format(base64string))
result = openurl(request, timeout=20)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(result.read()))
if response_code == 200 and response is not None:
Logger.info('Ambari admin user creation successful.')
return response_code
else:
Logger.info('Ambari admin user creation failed.')
return None
else:
return None
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason))
def create_ambari_admin_user(self, ambari_admin_username,
ambari_admin_password, usernamepassword):
try:
url = self.urlUsers + '?name=' + str(ambari_admin_username)
request = urllib2.Request(url)
base64string = base64.encodestring(usernamepassword).replace(
'\n', '')
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/json")
request.add_header("Authorization",
"Basic {0}".format(base64string))
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(result.read())
if response_code == 200 and len(response['vXUsers']) >= 0:
ambari_admin_username = ambari_admin_username
flag_ambari_admin_present = False
for vxuser in response['vXUsers']:
rangerlist_username = vxuser['name']
if rangerlist_username == ambari_admin_username:
flag_ambari_admin_present = True
break
else:
flag_ambari_admin_present = False
if flag_ambari_admin_present:
Logger.info(
ambari_admin_username +
' user already exists, using existing user from configurations.'
)
return ambari_admin_username, ambari_admin_password
else:
Logger.info(
ambari_admin_username +
' user is not present, creating user using given configurations'
)
url = self.urlSecUsers
admin_user = dict()
admin_user['status'] = 1
admin_user['userRoleList'] = ['ROLE_SYS_ADMIN']
admin_user['name'] = ambari_admin_username
admin_user['password'] = ambari_admin_password
admin_user['description'] = ambari_admin_username
admin_user['firstName'] = ambari_admin_username
data = json.dumps(admin_user)
base64string = base64.encodestring(
'{0}'.format(usernamepassword)).replace('\n', '')
headers = {
'Accept': 'application/json',
"Content-Type": "application/json"
}
request = urllib2.Request(url, data, headers)
request.add_header("Authorization",
"Basic {0}".format(base64string))
result = urllib2.urlopen(request)
response_code = result.getcode()
response = json.loads(json.JSONEncoder().encode(
result.read()))
if response_code == 200 and response is not None:
Logger.info('Ambari admin user creation successful.')
else:
Logger.info(
'Ambari admin user creation failed,setting username and password as blank'
)
ambari_admin_username = ''
ambari_admin_password = ''
return ambari_admin_username, ambari_admin_password
else:
return '', ''
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
Logger.error("HTTP Code: {0}".format(e.code))
Logger.error("HTTP Data: {0}".format(e.read()))
return '', ''
else:
Logger.error("Error: {0}".format(e.reason))
return '', ''