def verify_Policy_Exists_after_replication(
cls,
servicetype,
verify_from_cluster=source_weburl,
custer_to_verify=target_weburl,
database=None,
path=None,
NoPolicyInTarget=False,
expectedDesc="created by beacon while importing from " +
primaryCluster,
preDenyPolicyStr=primaryCluster + "_beacon deny policy for "):
if Xa.isArgusInstalled() is True:
serviceName = "hadoop" if servicetype == "hdfs" else servicetype
serviceNameOfverify_from_cluster = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True,
ambariWeburl=verify_from_cluster)[0]['name']
serviceNameOfverify_to_cluster = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=servicetype, status=True,
ambariWeburl=custer_to_verify)[0]['name']
logger.info("verifying if policy exist in target cluster")
policies_in_source_Cluster = Xa.getPoliciesForResources(
servicetype,
serviceName=serviceNameOfverify_from_cluster,
ambariWeburl=verify_from_cluster,
database=database,
path=path)
policies_in_target_Cluster = Xa.getPoliciesForResources(
servicetype,
serviceName=serviceNameOfverify_to_cluster,
ambariWeburl=custer_to_verify,
database=database,
path=path)
if NoPolicyInTarget == False:
assert len(policies_in_target_Cluster
) != 0, "make sure policies were imported"
BeaconRanger.setIdOfAllPolicyToZero(
policies_in_source_Cluster, policies_in_target_Cluster,
expectedDesc)
logger.info("set of policies in target cluster: " +
str(policies_in_target_Cluster["policies"]))
for policy in policies_in_source_Cluster["policies"]:
logger.info("policy is " + str(policy))
assert policy in policies_in_target_Cluster["policies"]
logger.info(
"all policies are verified!! now will check for deny policy if it is true"
)
isDenyPolicyTrue = Ambari.getConfig(
'beacon-security-site',
webURL=source_weburl)['beacon.ranger.plugin.create.denypolicy']
all_policies_in_target_Cluster = Xa.getPolicy_api_v2(
servicetype, weburl=target_weburl)
if isDenyPolicyTrue == 'true':
dataset = path if servicetype == "hdfs" else database
BeaconRanger.denyPolicyValidation(
servicetype, dataset, all_policies_in_target_Cluster,
preDenyPolicyStr)
else:
assert len(policies_in_target_Cluster) == len(
policies_in_source_Cluster)
def createPoliciesFromJson(cls,
file,
serviceType,
sourceHiveServiceName="mycluster0_hive",
sourceHdfsServiceName="mycluster0_hadoop",
targetServiceName=None,
ambariWeburl=source_weburl,
updateIfExists=False,
polResource=None,
isOverRideTrue=True):
if Xa.isArgusInstalled():
servicesMapJson = Config.getEnv(
'ARTIFACTS_DIR') + '/' + datetime.datetime.now().strftime(
"%Y%m%d%H%M%S") + 'service_mapping.json'
serviceName = "hadoop" if serviceType == "hdfs" else serviceType
if targetServiceName is None:
targetServiceName = \
Xa.findRepositories(nameRegex="^.*_" + serviceName + "$", type=serviceType, status=True,
ambariWeburl=ambariWeburl)[0]['name']
f = open(servicesMapJson, 'w')
if serviceType == "hive":
f.write('{"' + sourceHiveServiceName + '":"' +
targetServiceName + '"}')
elif serviceType == "hdfs":
f.write('{"' + sourceHdfsServiceName + '":"' +
targetServiceName + '"}')
f.close()
Xa.importPoliciesInJsonFile(file,
serviceType,
servicesMapJson=servicesMapJson,
ambariWeburl=ambariWeburl,
updateIfExists=updateIfExists,
polResource=polResource,
isOverRideTrue=isOverRideTrue)
def deleteRangerPolicyBasedOndDatabase(cls,
serviceType,
db,
serviceName=None,
weburl=None,
deleteOnlyDenyPolicies=False):
if serviceName is None:
repos = Xa.findRepositories(nameRegex="^.*_" + serviceType + "$",
type=serviceType,
status=True,
ambariWeburl=weburl)
serviceName = repos[0]['name']
policies_to_delete = Xa.getPoliciesForResources(serviceType,
serviceName,
database=db,
ambariWeburl=weburl)
if policies_to_delete is not None:
for policy in policies_to_delete["policies"]:
if deleteOnlyDenyPolicies == True:
if primaryCluster + "_beacon deny policy for " + db == policy[
"name"]:
Xa.deletePolicy_by_id_api_v2(policy["id"],
weburl=weburl)
break
else:
Xa.deletePolicy_by_id_api_v2(policy["id"], weburl=weburl)
#waiting for policy refresh after policies deletion
time.sleep(30)
def getRangerConfigs(cls, ambariWeburl=None):
if Xa.isArgusInstalled() is True:
logger.info("Ranger is ON")
Address = Xa.getPolicyAdminAddress(ambariWeburl=ambariWeburl)
hadoop_repo = \
Xa.findRepositories(nameRegex="^.*_" + "hadoop" + "$", type="hdfs", status=True, ambariWeburl=ambariWeburl)[
0]['name']
hive_repo = \
Xa.findRepositories(nameRegex="^.*_" + "hive" + "$", type="hive", status=True, ambariWeburl=ambariWeburl)[
0]['name']
ranger_config = {
"ranger_url": Address,
"hadoop_repo": hadoop_repo,
"hive_repo": hive_repo
}
return ranger_config
else:
return None
def setupOpenRangerHivePolicy(cls):
logger.info(
"============================== %s.%s ============================="
% (__name__, sys._getframe().f_code.co_name))
logger.info("setupOpenRangerHivePolicy: Begin")
repos = Xa.findRepositories(nameRegex="^.*_hive$",
type="Hive",
status=True)
if len(repos) == 0:
repo = {}
repo['repositoryType'] = 'Hive'
repo['name'] = "%s%d" % ('knox_test_hive_repo_', time.time())
repo['description'] = 'Knox Test Hive Repo'
repo['version'] = '0.4.0.2.2.2.0-2509'
repo['isActive'] = True
config = {}
config['username'] = '******'
config['password'] = '******'
config['jdbc.driverClassName'] = 'org.apache.hive.jdbc.HiveDriver'
config[
'jdbc.url'] = 'jdbc:hive2://ip-172-31-37-219.ec2.internal:10000/default;principal=hive/[email protected]'
config['commonNameForCertificate'] = ''
config['isencrypted'] = True
repo = Xa.createPolicyRepository(repo, config)
else:
assert len(
repos
) == 1, "Found wrong number of Hive Ranger policy repos. Expected 1, found %d." % len(
repos)
repo = repos[0]
#print "REPO=" + jsonlib.dumps(repo,indent=4)
t = time.time()
policy = {}
policy['repositoryName'] = repo['name']
policy['repositoryType'] = repo['repositoryType']
policy['policyName'] = "%s%s%d" % (repo['name'],
'_open_public_test_policy_', t)
policy['description'] = 'Open Knox Public Test Policy'
policy['databases'] = '*, default'
policy['tables'] = "*,%d" % t
policy['columns'] = "*,%d" % t
policy['isEnabled'] = True
policy['isAuditEnabled'] = True
policy['tableType'] = 'Inclusion'
policy['columnType'] = 'Inclusion'
policy['permMapList'] = {
'groupList': ['public'],
'permList': [
'select', 'update', 'create', 'drop', 'alter', 'index', 'lock',
'all', 'admin'
]
},
#print "CREATE=" + jsonlib.dumps(policy)
result = Xa.createPolicy(policy)
logger.info("setupOpenRangerHivePolicy: %s" % jsonlib.dumps(result))
return result
def setupOpenRangerKnoxPolicy(cls):
logger.info(
"============================== %s.%s ============================="
% (__name__, sys._getframe().f_code.co_name))
logger.info("setupOpenRangerKnoxPolicy: Begin")
repos = Xa.findRepositories(nameRegex="^.*_knox$",
type="Knox",
status=True)
if len(repos) == 0:
repo = {}
repo['repositoryType'] = 'Knox'
repo['name'] = "%s%d" % ('knox_test_knox_repo_', time.time())
repo['description'] = 'Knox Test Knox Repo'
repo['version'] = '0.1.0'
repo['isActive'] = True
config = {}
config['username'] = Knox.getAdminUsername()
config['password'] = Knox.getAdminPassword()
config[
'knox.url'] = 'https://%KNOX_HOST%:8443/gateway/admin/api/v1/topologies'
config['commonNameForCertificate'] = ''
repo = Xa.createPolicyRepository(repo, config)
else:
assert len(
repos
) == 1, "Found wrong number of Knox Ranger policy repos. Expected 1, found %d." % len(
repos)
repo = repos[0]
t = time.time()
policy = {}
policy['repositoryName'] = repo['name']
policy['repositoryType'] = repo['repositoryType']
policy['policyName'] = "%s%s%d" % (repo['name'],
'_open_public_test_policy_', t)
policy['description'] = 'Knox Open Public Test Policy'
policy['topologies'] = "*,%d" % t
policy['services'] = "*,%d" % t
policy['isEnabled'] = True
policy['isRecursive'] = True
policy['isAuditEnabled'] = True
policy['permMapList'] = [{
'groupList': ['public'],
'permList': ['allow']
}]
#print "CREATE=" + jsonlib.dumps(policy)
result = Xa.createPolicy(policy)
#print "CREATED=" + jsonlib.dumps(result)
logger.info("setupOpenRangerKnoxProxy: %s" % jsonlib.dumps(result))
return result