def execute(system_requests, machine_config, user, no_of_execution):
for i in range(0, no_of_execution):
security_enabled = machine_config.get('cluster', 'security', False)
yarn_jar = machine_config.get(
'yarn', 'yarn_jar',
'/usr/hdp/current/hadoop-mapreduce-client/hadoop-mapreduce-examples.jar'
)
queue_name = machine_config.get('yarn', 'queue_name', 'default')
user = machine_config.get(user, 'user', user)
principal = machine_config.get(user, 'principal', None)
keytab = machine_config.get(user, 'keytab', None)
sleep_time = machine_config.get('yarn', 'sleep_time', '3')
kinit_command = ""
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1};".format(
keytab, principal)
yarn_command = "{0} yarn jar {1} pi -Dmapred.job.queue.name={2} 2 2".format(
kinit_command, yarn_jar, queue_name)
code, stdout = system_requests.execute_command(yarn_command, user)
AuditLogger.info("YARN COMMAND RESULT FOR USER - " + str(user) +
" : " + str(stdout))
time.sleep(int(sleep_time))
def execute(system_requests, machine_config, user, no_of_execution):
for i in range(0, no_of_execution):
security_enabled = machine_config.get('cluster', 'security', False)
hbase_cmd = machine_config.get('hbase', 'hbase_cmd', '/usr/hdp/current/hbase-client/bin/hbase')
user = machine_config.get(user, 'user', user)
principal = machine_config.get(user, 'principal', None)
keytab = machine_config.get(user, 'keytab', None)
random_numbers = random.randint(100000000000,999999999999)
sleep_time = machine_config.get('hbase', 'sleep_time', '10')
kinit_command = ""
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1};".format(keytab, principal)
hbase_create_command = "create 'test_table_{0}', 'col1', 'col2'".format(random_numbers)
hbase_disable_command = "disable 'test_table_{0}'".format(random_numbers)
hbase_drop_command = "drop 'test_table_{0}'".format(random_numbers)
temp_file_path = "/tmp/test_tmp_{0}".format(random_numbers)
temp_file = open(temp_file_path, 'w')
temp_file.write(hbase_create_command + "\n" + hbase_disable_command + "\n" + hbase_drop_command + "\nexit\n")
temp_file.close()
hbase_command = "{0} {1} shell {2}".format(kinit_command, hbase_cmd, temp_file_path)
code, stdout = system_requests.execute_command(hbase_command, user)
AuditLogger.info("HBASE COMMAND RESULT : " + str(stdout))
time.sleep(int(sleep_time))
os.remove(temp_file_path)
def execute(system_requests, machine_config, no_of_execution):
user = machine_config.get('ambari-infra', 'user', 'infra-solr')
security_enabled = machine_config.get('cluster', 'security', False)
principal = machine_config.get('ambari-infra', 'principal', None)
keytab = machine_config.get('ambari-infra', 'keytab', None)
solr_server_address = machine_config.get('ambari-infra',
'solr_server_address', None)
solr_ranger_collection = machine_config.get('ambari-infra',
'solr_ranger_collection', None)
document_number = int(
machine_config.get('ambari-infra', 'document_number', '100'))
ip_address = machine_config.get('host', 'ip_address', None)
cluster = machine_config.get('cluster', 'cluster_name', None)
sleep_time = int(machine_config.get('ambari-infra', 'sleep_time', '10'))
for i in range(0, no_of_execution):
curl_prefix = 'curl -k'
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1}".format(
keytab, principal)
code, stdout = system_requests.execute_command(kinit_command, user)
AuditLogger.info("KINIT COMMAND RESULT : " + str(stdout))
curl_prefix = 'curl -k --negotiate -u :'
curl = '{0} -X POST -H \'Content-Type: application/json\''.format(
curl_prefix)
url = '{0}/solr/{1}/update'.format(solr_server_address,
solr_ranger_collection)
data = '['
for i in range(0, document_number):
if i > 0:
data += ','
random_numbers = random.randint(100000000000, 999999999999)
now = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')
data += '{{ "id": "{0}", "access": "WRITE", "enforcer": "ranger-acl", "repo": "{3}_fill", "reqUser": "******", "resource": "resource {0}", "cliIP": "{1}", "logType": "RangerAudit", "result": 1, "policy": 2, "repoType": 1, "resType": "path", "reason": "reason {0}", "action": "write", "evtTime": "{2}", "seq_num": 0, "event_count": 1, "event_dur_ms": 0, "cluster": "{3}" }}'.format(
random_numbers, ip_address, now, cluster)
data += ']'
curl_command = '{0} \'{1}\' --data-binary \'{2}\''.format(
curl, url, data)
code, stdout = system_requests.execute_command(curl_command, user, 200)
AuditLogger.info("FILL COMMAND RESULT : " + str(stdout))
time.sleep(sleep_time)
def execute(system_requests, machine_config, user, no_of_execution):
for i in range(0, no_of_execution):
security_enabled = machine_config.get('cluster', 'security', False)
user = machine_config.get(user, 'user', user)
principal = machine_config.get(user, 'principal', None)
keytab = machine_config.get(user, 'keytab', None)
hive_jdbc_url = machine_config.get('hive', 'hive_jdbc_url', None)
database = machine_config.get('hive', 'database', 'default')
random_numbers = random.randint(100000000000,999999999999)
sleep_time = machine_config.get('hive', 'sleep_time', '10')
kinit_command = ""
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1};".format(keytab, principal)
hive_command = "{0} /usr/bin/beeline -u '{1}' -n {2} -p {3} -d org.apache.hive.jdbc.HiveDriver -e 'use {4};'".format(kinit_command, hive_jdbc_url, user, user, database)
code, stdout = system_requests.execute_command(hive_command, user)
time.sleep(int(sleep_time))
AuditLogger.info("HIVE COMMAND RESULT FOR USE DATABASE NAME" + str(database) + " : " + str(stdout))
hive_command = "{0} /usr/bin/beeline -u '{1}' -n {2} -p {3} -d org.apache.hive.jdbc.HiveDriver -e 'create table {4}.test_table_{5}(id Int, name String);'".format(kinit_command, hive_jdbc_url, user, user, database, random_numbers)
code, stdout = system_requests.execute_command(hive_command, user)
time.sleep(int(sleep_time))
AuditLogger.info("HIVE COMMAND RESULT FOR CREATE TABLE NAME test_table_" + str(random_numbers) + " : " + str(stdout))
hive_command = "{0} /usr/bin/beeline -u '{1}' -n {2} -p {3} -d org.apache.hive.jdbc.HiveDriver -e 'drop table if exists {4}.test_table_{5};'".format(kinit_command, hive_jdbc_url, user, user, database, random_numbers)
code, stdout = system_requests.execute_command(hive_command, user)
time.sleep(int(sleep_time))
AuditLogger.info("HIVE COMMAND RESULT FOR DROP TABLE NAME test_table_" + str(random_numbers) + " : " + str(stdout))
hive_command = "{0} /usr/bin/beeline -u '{1}' -n {2} -p {3} -d org.apache.hive.jdbc.HiveDriver -e 'show databases;'".format(kinit_command, hive_jdbc_url, user, user)
code, stdout = system_requests.execute_command(hive_command, user)
time.sleep(int(sleep_time))
AuditLogger.info("HIVE COMMAND RESULT FOR SHOW DATABASES" + str(database) + " : " + str(stdout))
def execute(system_requests, machine_config, user, service, no_of_execution):
for i in range(0, no_of_execution):
user = machine_config.get(user, 'user', user)
password = machine_config.get(user, 'password', user + '-password')
knox_host = machine_config.get('knox', 'knox_host', 'localhost')
knox_port = machine_config.get('knox', 'knox_port', '8443')
knox_topology = machine_config.get('knox', 'knox_topology', 'default')
operation = "webhdfs/v1?op=LISTSTATUS"
if service == "WEBHDFS":
operation = "{0}/v1?op=LISTSTATUS".format(service.lower())
if service == "WEBHCAT":
operation = "templeton/v1/status"
knox_command = 'curl -i -k -u {0}:{1} -X GET https://{2}:{3}/gateway/{4}/{5}'.format(
user, password, knox_host, knox_port, knox_topology, operation)
code, stdout = system_requests.execute_command(knox_command, 'root')
AuditLogger.info("KNOX COMMAND RESULT : " + str(stdout))
def execute(system_requests, machine_config, user, no_of_execution):
for i in range(0, no_of_execution):
security_enabled = machine_config.get('cluster', 'security', False)
user = machine_config.get(user, 'user', user)
principal = machine_config.get(user, 'principal', None)
keytab = machine_config.get(user, 'keytab', None)
key_create = "hadoop key create"
key_list = "hadoop key list"
key_rollover = "hadoop key roll"
ranger_kms_service = machine_config.get('ranger_kms',
'ranger_kms_service',
'cl1_kms')
ranger_host = machine_config.get('ranger', 'ranger_host', 'localhost')
ranger_port = machine_config.get('ranger', 'ranger_port', '6080')
ranger_url = "http://{0}:{1}".format(ranger_host, ranger_port)
random_numbers = random.randint(100000000000, 999999999999)
kinit_command = ""
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1};".format(
keytab, principal)
key_create_command = '{0} {1} test_key_{2}'.format(
kinit_command, key_create, random_numbers)
code, stdout = system_requests.execute_command(key_create_command,
user)
AuditLogger.info("RANGER KMS COMMAND RESULT FOR CREATE KEY test_key_" +
str(random_numbers) + " : " + str(stdout))
key_list_command = '{0} {1}'.format(kinit_command, key_list)
code, stdout = system_requests.execute_command(key_list_command, user)
AuditLogger.info("RANGER KMS COMMAND RESULT FOR CREATE KEY test_key_" +
str(random_numbers) + " : " + str(stdout))
key_rollover_command = '{0} {1} test_key_{2}'.format(
kinit_command, key_rollover, random_numbers)
code, stdout = system_requests.execute_command(key_rollover_command,
user)
AuditLogger.info(
"RANGER KMS COMMAND RESULT FOR ROLLOVER KEY test_key_" +
str(random_numbers) + " : " + str(stdout))
key_delete_command = 'curl -i -u keyadmin:keyadmin -X DELETE {0}/service/keys/key/test_key_{1}?provider={2}'.format(
ranger_url, random_numbers, ranger_kms_service)
code, stdout = system_requests.execute_command(key_delete_command,
'root')
AuditLogger.info("RANGER KMS COMMAND RESULT FOR DELETE KEY test_key_" +
str(random_numbers) + " : " + str(stdout))
def execute_command(self, command, user, max_log_length=sys.maxint):
env = os.environ.copy()
subprocess_command = [
"su", user, "-l", "-s", "/bin/bash", "-c", command
]
log_subprocess_command = " ".join(subprocess_command)
log_subprocess_command = log_subprocess_command if len(
log_subprocess_command
) < max_log_length else log_subprocess_command[:max_log_length] + '...'
AuditLogger.info("RUNNING COMMAND: " + log_subprocess_command)
proc = subprocess.Popen(subprocess_command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=False,
cwd=None,
env=env)
result = proc.communicate()
proc_stdout = result[0]
proc_returncode = proc.returncode
return proc_returncode, proc_stdout
def execute(system_requests, machine_config, user, no_of_execution):
for i in range(0, no_of_execution):
security_enabled = machine_config.get('cluster', 'security', False)
user = machine_config.get(user, 'user', user)
principal = machine_config.get(user, 'principal', None)
keytab = machine_config.get(user, 'keytab', None)
random_numbers = random.randint(100000000000, 999999999999)
sleep_time = machine_config.get('hdfs', 'sleep_time', '10')
kinit_command = ""
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1};".format(
keytab, principal)
hdfs_ls_command = "{0} hdfs dfs -ls /tmp | head -n 1".format(
kinit_command)
code, stdout = system_requests.execute_command(hdfs_ls_command, user)
time.sleep(int(sleep_time))
AuditLogger.info("HDFS COMMAND RESULT FOR LIST FILES : " +
str(stdout) + " ...")
hdfs_mkdir_command = "{0} hdfs dfs -mkdir -p /tmp/test_dir_{1}".format(
kinit_command, random_numbers)
code, stdout = system_requests.execute_command(hdfs_mkdir_command,
user)
time.sleep(int(sleep_time))
AuditLogger.info("HDFS COMMAND RESULT FOR MAKE DIRECTORY : " +
str(stdout))
hdfs_ls_dir_command = "{0} hdfs dfs -ls /tmp/test_dir_{1}".format(
kinit_command, random_numbers)
code, stdout = system_requests.execute_command(hdfs_ls_dir_command,
user)
time.sleep(int(sleep_time))
AuditLogger.info(
"HDFS COMMAND RESULT FOR LIST DIRECTORY /tmp/test_dir_" +
str(random_numbers) + " : " + str(stdout))
hdfs_remove_command = "{0} hdfs dfs -rm -r /tmp/test_dir_{1}".format(
kinit_command, random_numbers)
code, stdout = system_requests.execute_command(hdfs_remove_command,
user)
time.sleep(int(sleep_time))
AuditLogger.info(
"HDFS COMMAND RESULT FOR DELETE DIRECTORY /tmp/test_dir_" +
str(random_numbers) + " : " + str(stdout))
def execute(system_requests, machine_config, user, no_of_execution):
for i in range(0, no_of_execution):
security_enabled = machine_config.get('cluster', 'security', False)
user = machine_config.get(user, 'user', user)
principal = machine_config.get(user, 'principal', None)
keytab = machine_config.get(user, 'keytab', None)
kafka_home = machine_config.get('kafka', 'kafka_home',
'/usr/hdp/current/kafka-broker')
zookeeper_quorum = machine_config.get('zookeeper', 'zookeeper_quorum',
'localhost:2181')
kafka_create_topic = "{0}/bin/kafka-topics.sh --create --zookeeper {1} --replication-factor 1 --partitions 1 --topic".format(
kafka_home, zookeeper_quorum)
kafka_list_topic = "{0}/bin/kafka-topics.sh --list --zookeeper {1}".format(
kafka_home, zookeeper_quorum)
kafka_delete_topic = "{0}/bin/kafka-topics.sh --delete --zookeeper {1} --topic".format(
kafka_home, zookeeper_quorum)
random_numbers = random.randint(100000000000, 999999999999)
sleep_time = machine_config.get('kafka', 'sleep_time', '10')
kinit_command = ""
if security_enabled == 'True':
kinit_command = "/usr/bin/kinit -kt {0} {1};".format(
keytab, principal)
kafka_create_command = '{0} {1} test_topic_{2}'.format(
kinit_command, kafka_create_topic, random_numbers)
code, stdout = system_requests.execute_command(kafka_create_command,
user)
time.sleep(int(sleep_time))
AuditLogger.info("KAFKA COMMAND RESULT FOR CREATE TOPIC test_topic_" +
str(random_numbers) + " : " + str(stdout))
kafka_list_command = '{0} {1}'.format(kinit_command, kafka_list_topic)
code, stdout = system_requests.execute_command(kafka_list_command,
user)
time.sleep(int(sleep_time))
AuditLogger.info("KAFKA COMMAND RESULT FOR LIST TOPIC test_topic_" +
str(random_numbers) + " : " + str(stdout))
kafka_delete_command = '{0} {1} test_topic_{2}'.format(
kinit_command, kafka_delete_topic, random_numbers)
code, stdout = system_requests.execute_command(kafka_delete_command,
user)
time.sleep(int(sleep_time))
AuditLogger.info("KAFKA COMMAND RESULT FOR DELETE TOPIC test_topic_" +
str(random_numbers) + " : " + str(stdout))
def __init__(self, type):
AuditLogger.initialize_logger(type)
AuditLogger.info("Initializing Logger for Ranger Audits [" + type +
"]")