def __init__(self, config):
self.config = config
self.logger = logging.getLogger('ClouderaManagerMetricsCollector')
self._setup_logging(config)
self.logger.info("Load configuration: " + str(config))
self.cluster = None
self.services = []
self.availableMetrics = {}
self.serviceMetricsMapFromConfig = {}
self._create_service_metrics_map()
self._setup_termination_handler()
self._setup_metrics_processor(config)
self._setup_health_check_metrics_processor(config)
cm_client.configuration.username = config["clouderaManager"]["user"]
cm_client.configuration.password = config["clouderaManager"][
"password"]
cm_client.configuration.verify_ssl = False
self.cluster_type = config["clouderaManager"]["clusterType"]
if socket.gethostname().find('.') >= 0:
api_host = socket.gethostname()
else:
api_host = socket.gethostbyaddr(socket.gethostname())[0]
protocol = config["clouderaManager"]["protocol"]
port = config["clouderaManager"]["port"]
api_version = config["clouderaManager"]["apiVersion"]
api_url = protocol + "://" + api_host + ':' + port + '/api/' + api_version
self.api_client = cm_client.ApiClient(api_url)
self.logger.info("Cloudera Manager Api URL: " + api_url)
metrics_api_url = protocol + "://" + api_host + ':' + port + '/api/v5'
self.metrics_api_client = cm_client.ApiClient(metrics_api_url)
self.logger.info("Cloudera Manager Metrics Api URL: " +
metrics_api_url)
self.globalFields = config["globalFields"]
def __init__(self):
import cm_client as cm
cm.configuration.username = "******"
cm.configuration.password = "******"
self.api_client = cm.ApiClient(self.api_url)
self.services_api_health = cm.ServicesResourceApi(self.api_client)
self.services_api_IO = cm.TimeSeriesResourceApi(self.api_client)
def api_client(self):
if self._api_client is None:
if cm_client.configuration.ssl_ca_cert:
api_url = "https://" + self.host + ":7183/api"
else:
api_url = "http://" + self.host + ":7180/api"
self._api_client = cm_client.ApiClient(api_url + '/' + self._get_api_version())
return self._api_client
def cluster_stats():
"""Gets the total YARN meory & VCores using the Cloudera anager API"""
#req = requests.get("http://nightly6x-unsecure-1.nightly6x-unsecure.root.hwx.site:7180/api/v40/tools/echo?message=hello")
#print req.content
cm_client.configuration.username = '******'
cm_client.configuration.password = '******'
# Create an instance of the API class
api_host = 'http://nightly6x-unsecure-1.nightly6x-unsecure.root.hwx.site'
port = '7180'
api_version = 'v33'
# Construct base URL for API
# http://cmhost:7180/api/v30
api_url = api_host + ':' + port + '/api/' + api_version
print api_url
api_client = cm_client.ApiClient(api_url)
cluster_api_instance = cm_client.ClustersResourceApi(api_client)
api_response = cluster_api_instance.read_clusters(view='SUMMARY')
for cluster in api_response.items:
print cluster.name, "-", cluster.full_version
services_api_instance = cm_client.ServicesResourceApi(api_client)
services = services_api_instance.read_services(cluster.name, view='FULL')
for service in services.items:
if service.type=="YARN":
yarn = service
print yarn.name
api_url_v5 = api_host + '/api/' + 'v5'
api_client_v5 = cm_client.ApiClient(api_url_v5)
print api_url_v5
services_api_instance_v5 = cm_client.ServicesResourceApi(api_client_v5)
#print services_api_instance_v5.get_metrics(cluster.name, hdfs.name)
metrics = services_api_instance_v5.get_metrics(cluster.name, yarn.name)
for m in metrics.items:
print "%s (%s)" % (m.name, m.unit)
def init_conn(self, host, port, version):
"""初始化连接，构建cm的连接实例
"""
url = host + ':' + port + '/api/' + version
client = cm_client.ApiClient(url)
# 生成资源API
self._cluster_api_instance = cm_client.ClustersResourceApi(client)
# 为了获取namemode的资源API
self._services_api_instance = cm_client.ServicesResourceApi(client)
self._roles_api_instance = cm_client.RolesResourceApi(client)
self._host_api_instance = cm_client.HostsResourceApi(client)
def main():
cm_client.configuration.username = ADMIN_USER
cm_client.configuration.password = ADMIN_PASS
# Create an instance of the API class
api_url = "http://{}:{}/api/{}".format(CM_HOST, CM_PORT, API_VERSION)
api_client = cm_client.ApiClient(api_url)
api_instance = cm_client.ClouderaManagerResourceApi(api_client)
try:
setup_logger.info("Activating Trial License...")
api_instance.begin_trial()
setup_logger.info("License Activated...")
except:
setup_logger.error("Failed to Activate License...")
def cm_kafkaXX1_host_command(update: Update, context: CallbackContext) -> None:
chat_ids = update.message.chat_id
api_client = cm_client.ApiClient(api_url_dc)
api_instance = cm_client.ClustersResourceApi(api_client)
api_response = api_instance.list_hosts('kafkaXX1')
print(api_response)
mo = ('Cluster: <b>kafkaXX1</b>:\n')
for cluster in api_response.items:
mo = mo + ('Host: ' + cluster.hostname + '\n')
context.bot.send_message(chat_id=chat_ids,
parse_mode=ParseMode.HTML,
text=(mo))
def cm_drc_cluster_command(update: Update, context: CallbackContext) -> None:
chat_ids = update.message.chat_id
api_client = cm_client.ApiClient(api_url_drc)
cluster_api_instance = cm_client.ClustersResourceApi(api_client)
api_response = cluster_api_instance.read_clusters(view='full')
for cluster in api_response.items:
mo = ('Site: <b>DRC</b>:\n')
mo = mo + ('Name: <b>' + cluster.name + '</b>\n')
mo = mo + ('Version: ' + cluster.full_version + '\n')
mo = mo + ('Status: ' + cluster.entity_status + '\n')
mo = mo + ('Maintenance: ' + str(cluster.maintenance_mode) + '\n')
mo = mo + ('URL: ' + cluster.cluster_url + '\n')
context.bot.send_message(chat_id=chat_ids,
parse_mode=ParseMode.HTML,
text=(mo))
def main():
module = build_module()
choice_map = {
'present': 'downloaded',
'distributed': 'distributed',
'activated': 'activated',
'absent': 'available_remotely',
'infos': 'infos'
}
params = module.params
api_url = f"http://{params['cm_host']}:{params['cm_port']}/api/v{params['api_version']}"
cm_client.configuration.username = params['cm_login']
cm_client.configuration.password = params['cm_password']
cm_client.configuration.host = api_url
api_client = cm_client.ApiClient()
if params["product"] and params["version"]:
parcel = Parcel(params["product"], params["version"],
params["cluster_name"], api_client)
try:
getattr(parcel, choice_map.get(params["state"]))()
except ApiException as e:
module.fail_json(msg=f"Cluster error : {e}")
module.exit_json(changed=parcel.changed,
msg="Parcel informations gathered",
meta=parcel.meta())
else:
if params["state"] == "infos":
api_client_instance = cm_client.ParcelsResourceApi(api_client)
parcels = []
try:
for parcel in api_client_instance.read_parcels(
params["cluster_name"]).items:
if params["product"]:
if params["product"] != parcel.product:
continue
parcels.append(
Parcel(name=parcel.product,
version=parcel.version,
cluster_name=parcel.cluster_ref.cluster_name,
api_client=api_client).meta())
except ApiException as e:
module.fail_json(msg=f"Cluster error : {e}")
module.fail_json(
changed=False,
msg="No valid parameters combination was used, exiting",
meta=parcels)
def cm_clusterZ1_services_command(update: Update,
context: CallbackContext) -> None:
chat_ids = update.message.chat_id
api_client = cm_client.ApiClient(api_url_drc)
api_instance = cm_client.ServicesResourceApi(api_client)
api_response = api_instance.read_services('clusterZ1')
mo = ('Cluster: <b>clusterZ1</b>:\n')
for cluster in api_response.items:
if (cluster.health_summary == 'GOOD'):
mo = (mo + '\U00002705')
mo = mo + ('[<b>' + cluster.display_name + '</b>] => ' +
cluster.health_summary + '=> ' + cluster.service_state +
'\n')
context.bot.send_message(chat_id=chat_ids,
parse_mode=ParseMode.HTML,
text=(mo))
def __init__(self, cm_host, env, send_alert):
cm_client.configuration.username = '******'
cm_client.configuration.password = '******'
self.env = env
api_host = 'http://' + cm_host
port = '7180'
api_version = 'v19'
api_url = api_host + ':' + port + '/api/' + api_version
self.send_alert_email = send_alert
print(api_url)
self.api_client = cm_client.ApiClient(api_url)
self.cluster_api_instance = cm_client.ClustersResourceApi(
self.api_client)
self.services_api_instance = cm_client.ServicesResourceApi(
self.api_client)
self.roles_api_instance = cm_client.RolesResourceApi(self.api_client)
self.host_api_instance = cm_client.HostsResourceApi(self.api_client)
self.host_details = self.get_hostname_by_id()
self.bad_alert_count = 0
self.msg_body = ""
def __init__(self, instance):
self.log = logging.getLogger(__name__)
self.url, user, password, api_version, verify_ssl = get_config(
instance)
if not user:
raise Exception('Cloudera Manager user name is required.')
if not password:
raise Exception('Cloudera Manager user password is required.')
# Configure HTTP basic authorization: basic
cm_client.configuration.username = user
cm_client.configuration.password = password
cm_client.configuration.verify_ssl = verify_ssl
# Construct base URL for API
api_url = '{0}/api/{1}'.format(self.url, api_version)
self.api_client = cm_client.ApiClient(api_url)
def main():
cm_client.configuration.username = ADMIN_USER
cm_client.configuration.password = ADMIN_PASS
# Create an instance of the API class
api_host = 'http://'+ CM_HOST
port = '7180'
api_version = 'v18'
api_url = api_host + ':' + port + '/api/' + api_version
api_client = cm_client.ApiClient(api_url)
host_api_instance = cm_client.HostsResourceApi(api_client)
service_api_instance = cm_client.ServicesResourceApi(api_client)
sentry_host_id=fetch_hostid(HostsApi=host_api_instance, hostname=SENTRY_HA_HOST)
cluster_name = CLUSTER_NAME
service_name = "Sentry"
body = cm_client.ApiEnableSentryHaArgs(new_sentry_host_id=sentry_host_id, new_sentry_role_name= "Sentry-Server-2", zk_service_name="ZOOKEEPER")
setup_logger.info("Enabling Sentry HA.....")
time.sleep(60)
res=service_api_instance.enable_sentry_ha_command(cluster_name, service_name, body=body)
time.sleep(60)
setup_logger.info("Sentry HA Enabled...")
def main():
module = build_module()
params = module.params
api_url = f"http://{params['cm_host']}:{params['cm_port']}/api/v{params['api_version']}"
cm_client.configuration.username = params['cm_login']
cm_client.configuration.password = params['cm_password']
cm_client.configuration.host = api_url
api_client = cm_client.ApiClient()
cm_config = CM(name=params["cm_host"], api_client=api_client)
try:
cm_config.set_prop(name=params["name"],
state=params["action"],
value=params["value"])
except ApiException as e:
module.fail_json(msg=f"Cluster error : {e}")
module.exit_json(changed=cm_config.changed,
msg=f'{params["name"]} is in desired state',
meta=cm_config.meta())
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(description='Cloudera Cluster and Management Service stop commands')
parser.add_argument('configfile', nargs='?', default='config.yaml',
help='CM API access information in YAML (default config.yaml)')
args = parser.parse_args()
try:
with open(args.configfile, 'r') as f:
config = yaml.safe_load(f)
def config_value(key, default):
return config[key] if key in config else default
# Configure HTTP basic authentication: basic
cm_client.configuration.username = os.getenv('CM_ADMIN_USERNAME', config_value('username', 'admin'))
cm_client.configuration.password = os.getenv('CM_ADMIN_PASSWORD', config_value('password', 'admin'))
# Create an instance of the API class
protocol = config_value('protocol', 'http')
api_host = config_value('api_host', 'cloudera-manager.example.com')
api_port = config_value('api_port', '7180')
api_version = config_value('api_version', 'v32')
api_uri = f'{protocol}://{api_host}:{api_port}/api/{api_version}'
logger.debug(f'CM API URI: {api_uri}')
api_client = cm_client.ApiClient(api_uri)
stop_all_clusters(api_client, api_host)
stop_management_service(api_client, api_host)
return 0
except IOError as err:
logger.error(f'IOError: [Errno {err.errno}] {err.strerror}: {args.configfile}')
return -1
def get_api_handle(cmserver,
port='7180',
user='******',
passwd='admin',
api_version='v30'):
"""
Get a handle to the CM API client
:param cmserver: Hostname of the Cloudera Manager Server (CMS)
:param port: Port of the server
:param user: SCM username
:param passwd: SCM password
:param tls: Whether to use TLS
:return: Resource object referring to the root
"""
cm_client.configuration.username = user
cm_client.configuration.password = passwd
api_url = 'http://' + cmserver + ':' + port + '/api/' + api_version
api_client = cm_client.ApiClient(api_url)
api = None
try:
api = cm_client.HostsResourceApi(api_client)
except ApiException:
pass
return api
import cm_client
from cm_client.rest import ApiException
from collections import namedtuple
from pprint import pprint
import json
# Configure HTTP basic authorization: basic
cm_client.configuration.username = '******'
cm_client.configuration.password = '******'
api_url = "http://10.209.239.13:7180/api/v17"
api_client = cm_client.ApiClient(api_url)
# create an instance of the API class
cluster_name = 'Cluster 1' # str |
clusters_api_instance = cm_client.ClustersResourceApi(api_client)
template = clusters_api_instance.export(cluster_name)
# Following step allows python fields with under_score to map
# to matching camelCase name in the API model before writing to json file.
json_dict = api_client.sanitize_for_serialization(template)
with open('/home/tpc/hs/tpcxBB-pipeline/13sep-2019-cluster_template.json', 'w') as out_file:
json.dump(json_dict, out_file, indent=4, sort_keys=True)
def main(passed_username, passed_password, passed_database):
PEER_NAME = 'PRODUCTION' # Previously
TARGET_CLUSTER_NAME = 'DEV' # defined
SOURCE_CLUSTER_NAME = 'cluster' # at Experian
cm_host = 'br1andvhmn11.passporthealth.com'
cm_client.configuration.username = passed_username
# Ensure that password is quoted
cm_client.configuration.password = "******" + passed_password + "'"
sourceDatabase = passed_database
# Setup authentication for SSL
cm_client.configuration.verify_ssl = True
cm_client.configuration.ssl_ca_cert = '/opt/cloudera/security/pki/x509/truststore.pem'
# Create an instance of the API class
api_host = 'https://br1andvhmn11.passporthealth.com'
port = '7183'
api_version = 'v30'
impala_host = 'br1anprhsn02.passporthealth.com'
# Construct base URL for API
# http://cmhost:7180/api/v30
api_url = api_host + ':' + port + '/api/' + api_version
api_client = cm_client.ApiClient(api_url)
cluster_api_instance = cm_client.ClustersResourceApi(api_client)
# Lists all known clusters.
api_response = cluster_api_instance.read_clusters(view='SUMMARY')
for cluster in api_response.items:
print cluster.name, "-", cluster.full_version
services_api_instance = cm_client.ServicesResourceApi(api_client)
services = services_api_instance.read_services(cluster.name, view='FULL')
for service in services.items:
# print service.display_name, "-", service.type
if service.type == 'HIVE':
targetHive = service
targetCluster = cluster
print targetHive.name, targetHive.service_state, targetHive.health_summary
for health_check in targetHive.health_checks:
print health_check.name, "---", health_check.summary
# print "Source database = " + sourceDatabase
###show_statement = "'show tables in " + sourceDatabase +"'"
###streamOperand = "impala-shell -i " + impala_host + " -d default -k --ssl --ca_cert=/opt/cloudera/security/pki/x509/truststore.pem -q " + show_statement
###stream=os.popen(streamOperand)
###
###output=stream.readlines()
###lineno =0
###numtables = 0
###tablenames = []
###for line in output:
### if lineno <= 2: # skip heading lines
### pass
### elif line[0:3] == "+--": # skip last line
### pass
### else: # strip out tablename
### name = line[2:]
### blank = name.index(' ')
### tablenames.append(name[0:blank])
### numtables +=1
### lineno +=1
###print str(numtables) + " tables in database " + sourceDatabase
###for table in tablenames:
### print table
tablenames = []
tablenames.append("test")
tablenames.append("test2")
api_root = ApiResource(cm_host, username=passed_username, password=passed_password, use_tls=True)
PEER_NAME = 'PRODUCTION'
SOURCE_HDFS_NAME = 'hdfs'
TARGET_HDFS_NAME = 'hdfs'
SOURCE_HIVE_NAME = 'hive'
TARGET_HIVE_NAME = 'hive'
SOURCE_CLUSTER_NAME = 'cluster'
TARGET_CLUSTER_NAME = 'DEV'
TARGET_YARN_SERVICE = 'yarn'
# Setup for Hive replication
hive = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HIVE_NAME)
hive_args = ApiHiveReplicationArguments(None)
hdfs_args = ApiHdfsReplicationArguments(None) # Needed for replicating table data stored in HDFS
hive_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HIVE_NAME)
# Define tables to replicate
table_filters = []
table = ApiHiveTable(None)
for tab in tablenames:
table.database = (passed_database)
table.tableName = (tab)
table_filters = []
table_filters.append(table)
print "Replicating " + passed_database + "." + tab
hive_args.tableFilters = table_filters
hive_args.force = True # Overwrite existing tables
hive_args.replicateData = True # Replicate table data stored in HDFS
hdfs_args.skipChecksumChecks = True
hdfs_args.skipListingChecksumChecks = True
hdfs_args.preserveBlockSize = True
hdfs_args.preserveReplicationCount = True
hdfs_args.preservePermissions = True
# Define HDFS portion of the Hive replication as needed
hdfs_args.destinationPath = '/user/bob.marshall/repltest' # Argument? Path relative to servicename?
hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE
hdfs_args.userName = passed_username
hdfs_args.sourceUser = passed_username
hive_args.hdfsArguments = hdfs_args
start = datetime.datetime.now()
end = start + datetime.timedelta(days=1)
interval = "DAY"
numinterval = 1
pause = True
print "Creating Hive Replication Schedule"
schedule = hive.create_replication_schedule(start, end, interval, numinterval, pause, hive_args)
print "Starting Hive Replication"
cmd = hive.trigger_replication_schedule(schedule.id)
print "Waiting for completion"
cmd = cmd.wait()
print "Getting result"
result = hive.get_replication_schedule(schedule.id).history[0].hiveResult
print result
print "Cleanup... Remove Hive replication schedule"
sch = hive.delete_replication_schedule(schedule.id)
print sch
exit(0)
#scheds = hive.get_replication_schedules()
#sch = hive.delete_replication_schedule(162)
# Setup for HDFS replication
hdfs = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HDFS_NAME)
hdfs_args = ApiHdfsReplicationArguments(None)
hdfs_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HDFS_NAME)
hdfs_args.sourcePath = '/user/bob.marshall/repltest'
hdfs_args.destinationPath = '/user/bob.marshall/repltest'
hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE
hdfs_args.userName = args.username
hdfs_args.sourceUser = args.username
hdfs_args.preserveBlockSize = True
hdfs_args.preserveReplicationCount = True
hdfs_args.preservePermissions = True
hdfs_args.skipChecksumChecks = True
hdfs_args.skipListingChecksumChecks = True
start = datetime.datetime.now()
end = start + datetime.timedelta(days=1)
interval = "DAY"
numinterval = 1
pause = True
#schedule = hdfs.create_replication_schedule(start, end, interval, interval, pause, hdfs_args)
print "Creating HDFS Replication Schedule"
schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_args)
print "Starting HDFS Replication"
cmd = hdfs.trigger_replication_schedule(schedule.id)
print "Waiting for completion"
cmd = cmd.wait()
print "Getting result"
result = hdfs.get_replication_schedule(schedule.id).history[0].hdfsResult
print result
print "Cleanup... Remove HDFS replication schedule"
sch = hdfs.delete_replication_schedule(schedule.id)
print sch
#scheds = hdfs.get_replication_schedules()
#sch = hdfs.delete_replication_schedule(27)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Perform BDR jobs while getting around BDR limitations.')
parser.add_argument("username")
parser.add_argument("password")
parser.add_argument("database")
args = parser.parse_args()
main(args.username, args.password, args.database)
now = time.time()
if deadline < now:
return cmd
else:
time.sleep(min(SLEEP_SECS, deadline - now))
else:
time.sleep(SLEEP_SECS)
except ApiException as e:
print(
"Exception when calling ClouderaManagerResourceApi->import_cluster_template: %s\n"
% e)
cm_client.configuration.username = '******'
cm_client.configuration.password = '******'
api_client = cm_client.ApiClient("http://localhost:7180/api/v32")
cm_api = cm_client.ClouderaManagerResourceApi(api_client)
# accept trial licence
cm_api.begin_trial()
# Install CM Agent on host
with open("/root/myRSAkey", "r") as f:
key = f.read()
instargs = cm_client.ApiHostInstallArguments(
host_names=['YourHostname'],
user_name='root',
private_key=key,
cm_repo_url='https://archive.cloudera.com/cm6/6.3.0',
def api_client(self):
if self._api_client is None:
self._api_client = cm_client.ApiClient(
"http://localhost:7180/api/v32")
return self._api_client
def main(passed_username, passed_password, passed_database):
PEER_NAME = 'PRODUCTION' # Previously
TARGET_CLUSTER_NAME = 'DEV' # defined
SOURCE_CLUSTER_NAME = 'cluster' # at Experian
cm_host = 'br1andvhmn11.passporthealth.com'
cm_client.configuration.username = passed_username
cm_client.configuration.password = passed_password
sourceDatabase = passed_database
# Setup authentication for SSL
cm_client.configuration.verify_ssl = True
cm_client.configuration.ssl_ca_cert = '/opt/cloudera/security/pki/x509/truststore.pem'
# Create an instance of the API class
api_host = 'https://br1andvhmn11.passporthealth.com'
port = '7183'
api_version = 'v30'
#For testing, until firewall opened, pull tablenames from target cluster. We are therefore simulating an overwrite.
#For real, change impalahost to pull tablenames from source cluster.
# impala_host = 'br1anprhsn02.passporthealth.com'
impala_host = 'br1andvhmn02.passporthealth.com'
# Construct base URL for API
# http://cmhost:7180/api/v30
api_url = api_host + ':' + port + '/api/' + api_version
api_client = cm_client.ApiClient(api_url)
cluster_api_instance = cm_client.ClustersResourceApi(api_client)
# Lists all known clusters.
api_response = cluster_api_instance.read_clusters(view='SUMMARY')
for cluster in api_response.items:
print cluster.name, "-", cluster.full_version
services_api_instance = cm_client.ServicesResourceApi(api_client)
services = services_api_instance.read_services(cluster.name, view='FULL')
for service in services.items:
# print service.display_name, "-", service.type
if service.type == 'HIVE':
targetHive = service
targetCluster = cluster
print targetHive.name, targetHive.service_state, targetHive.health_summary
for health_check in targetHive.health_checks:
print health_check.name, "---", health_check.summary
# print "Source database = " + sourceDatabase
###show_statement = "'show tables in " + sourceDatabase +"'"
###streamOperand = "impala-shell -i " + impala_host + " -d default -k --ssl --ca_cert=/opt/cloudera/security/pki/x509/truststore.pem -q " + show_statement
###stream=os.popen(streamOperand)
###
###output=stream.readlines()
###lineno =0
###numtables = 0
###tablenames = []
###for line in output:
### if lineno <= 2: # skip heading lines
### pass
### elif line[0:3] == "+--": # skip last line
### pass
### else: # strip out tablename
### name = line[2:]
### blank = name.index(' ')
### tablenames.append(name[0:blank])
### numtables +=1
### lineno +=1
###print str(numtables) + " tables in database " + sourceDatabase
###for table in tablenames:
### print table
tablenames = []
tablenames.append('b05006')
tablenames.append('b05007_bgpr_cape_rename')
tablenames.append('b07001_bgpr_cape')
tablenames.append('b13008_bgpr_cape')
tablenames.append('c16001_bgpr_cape')
tablenames.append('c24070_bgpr_cape')
tablenames.append('cde2018_bg_a1')
tablenames.append('cde2018_bg_a2')
tablenames.append('cde2018_bg_a3')
tablenames.append('cde2018_bg_a4')
tablenames.append('cde2018_bg_b')
tablenames.append('cde2018_bg_c1')
tablenames.append('cde2018_bg_c2')
tablenames.append('cde2018_bg_c3')
tablenames.append('cde2018_bg_d')
tablenames.append('cons_exp_annual_cye2018_bg')
tablenames.append('cons_exp_annual_fyp2018_bg')
tablenames.append('cons_exp_avgannual_cye2018_bg')
tablenames.append('cons_exp_avgannual_fyp2018_bg')
tablenames.append('cons_exp_gifts_cye2018_bg')
tablenames.append('cons_exp_gifts_fyp2018_bg')
tablenames.append('cye2018_bg_a1')
tablenames.append('cye2018_bg_a2')
tablenames.append('cye2018_bg_a3')
tablenames.append('cye2018_bg_a4')
tablenames.append('cye2018_bg_b')
tablenames.append('cye2018_bg_c1')
tablenames.append('cye2018_bg_c2')
tablenames.append('cye2018_bg_c3')
tablenames.append('cye2018_bg_d')
tablenames.append('fyp2018_bg_a1')
tablenames.append('fyp2018_bg_a2')
tablenames.append('fyp2018_bg_a3')
tablenames.append('fyp2018_bg_a4')
tablenames.append('fyp2018_bg_b')
tablenames.append('fyp2018_bg_c1')
tablenames.append('fyp2018_bg_c2')
tablenames.append('fyp2018_bg_c3')
tablenames.append('fyp2018_bg_d')
api_root = ApiResource(cm_host, username=passed_username, password=passed_password, use_tls=True)
PEER_NAME = 'PRODUCTION'
SOURCE_HDFS_NAME = 'hdfs'
TARGET_HDFS_NAME = 'hdfs'
SOURCE_HIVE_NAME = 'hive'
TARGET_HIVE_NAME = 'hive'
SOURCE_CLUSTER_NAME = 'cluster'
TARGET_CLUSTER_NAME = 'DEV'
TARGET_YARN_SERVICE = 'yarn'
# Setup for Hive replication
hive = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HIVE_NAME)
hive_args = ApiHiveReplicationArguments(None)
hdfs_args = ApiHdfsReplicationArguments(None) # Needed for replicating table data stored in HDFS
hive_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HIVE_NAME)
# Define tables to replicate
table_filters = []
table = ApiHiveTable(None)
for tab in tablenames:
# table.database = (passed_database)
#
# Hardwire database name for 2020-11-24 replication
#
table.database = 'lake_consumerview'
table.tableName = (tab)
table_filters = []
table_filters.append(table)
print "Replicating " + passed_database + "." + tab
hive_args.tableFilters = table_filters
hive_args.force = True # Overwrite existing tables
hive_args.replicateData = True # Replicate table data stored in HDFS
hdfs_args.skipChecksumChecks = True
hdfs_args.skipListingChecksumChecks = True
hdfs_args.preserveBlockSize = True
hdfs_args.preserveReplicationCount = True
hdfs_args.preservePermissions = True
# Define HDFS portion of the Hive replication as needed
# hdfs_args.destinationPath = '/'
hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE
hdfs_args.userName = passed_username
hdfs_args.sourceUser = passed_username
hive_args.hdfsArguments = hdfs_args
start = datetime.datetime.now()
end = start + datetime.timedelta(days=1)
interval = "DAY"
numinterval = 1
pause = True
print "Creating Hive Replication Schedule"
schedule = hive.create_replication_schedule(start, end, interval, numinterval, pause, hive_args)
print "Starting Hive Replication"
cmd = hive.trigger_replication_schedule(schedule.id)
print "Waiting for completion"
cmd = cmd.wait()
print "Getting result"
result = hive.get_replication_schedule(schedule.id).history[0].hiveResult
print result
print "Cleanup... Remove Hive replication schedule"
sch = hive.delete_replication_schedule(schedule.id)
print sch
exit(0)
#scheds = hive.get_replication_schedules()
#sch = hive.delete_replication_schedule(162)
# Setup for HDFS replication
hdfs = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HDFS_NAME)
hdfs_args = ApiHdfsReplicationArguments(None)
hdfs_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HDFS_NAME)
hdfs_args.sourcePath = '/user/bob.marshall/repltest'
hdfs_args.destinationPath = '/user/bob.marshall/repltest'
hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE
hdfs_args.userName = args.username
hdfs_args.sourceUser = args.username
hdfs_args.preserveBlockSize = True
hdfs_args.preserveReplicationCount = True
hdfs_args.preservePermissions = True
hdfs_args.skipChecksumChecks = True
hdfs_args.skipListingChecksumChecks = True
start = datetime.datetime.now()
end = start + datetime.timedelta(days=1)
interval = "DAY"
numinterval = 1
pause = True
#schedule = hdfs.create_replication_schedule(start, end, interval, interval, pause, hdfs_args)
print "Creating HDFS Replication Schedule"
schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_args)
print "Starting HDFS Replication"
cmd = hdfs.trigger_replication_schedule(schedule.id)
print "Waiting for completion"
cmd = cmd.wait()
print "Getting result"
result = hdfs.get_replication_schedule(schedule.id).history[0].hdfsResult
print result
print "Cleanup... Remove HDFS replication schedule"
sch = hdfs.delete_replication_schedule(schedule.id)
print sch
if deadline is not None:
now = time.time()
if deadline < now:
return cmd
else:
time.sleep(min(SLEEP_SECS, deadline - now))
else:
time.sleep(SLEEP_SECS)
except ApiException as e:
print("Exception when calling ClouderaManagerResourceApi->import_cluster_template: %s\n" % e)
cm_client.configuration.username = '******'
cm_client.configuration.password = '******'
api_client = cm_client.ApiClient("http://cloudera:7180/api/v40")
cm_api = cm_client.ClouderaManagerResourceApi(api_client)
# accept trial licence
# m_api.begin_trial()
# Install CM Agent on host
with open ("/root/myRSAkey", "r") as f:
key = f.read()
instargs = cm_client.ApiHostInstallArguments(
host_names=['YourHostname'],
user_name='root',
private_key=key,
cm_repo_url='https://archive.cloudera.com/cm7/7.1.3/',
if sys.argv[2] != 'KILL':
print("the only valid second argument is \"KILL\"")
printUsageMessage()
quit(1)
else:
kill = True
# Lists all known clusters.
# Construct base URL for API
# http://cmhost:7180/api/v17
api_url = api_host + ':' + port + '/api/' + api_version
## Connect to CM
print("\nConnecting to Cloudera Manager at " + api_host + ":" + port)
## Get the IMPALA service
api_client = cm_client.ApiClient(api_url)
services_api_instance = cm_client.ServicesResourceApi(api_client)
services = services_api_instance.read_services(cluster_name, view='FULL')
for service in services.items:
#print(service.name, "-", service.type)
if service.type == "IMPALA":
impala_service = service
print("Located Impala Service: " + service.name)
break
if impala_service is None:
print("Error: Could not locate Impala Service")
quit(1)
## A window of one day assumes queries have not been running more than 24 hours
now = datetime.utcnow()
start = now - timedelta(days=1)
cm_client_username = args.user
cm_client_password = args.password
api_protocol = args.protocol
api_host = args.host
port = args.port
api_version = args.apiver
cluster = args.cluster
service = SERVICE
cm_client.configuration.username = cm_client_username
cm_client.configuration.password = cm_client_password
url = api_protocol + "://" + api_host + ":" + port + "/api/" + api_version
api_client = cm_client.ApiClient(url)
role_config_group_resource_api = cm_client.RoleConfigGroupsResourceApi(
api_client)
config_by_roles = role_config_group_resource_api.read_role_config_groups(
cluster_name=cluster, service_name=service)
config_by_group = {}
role_config_group_name_displayname_mapping = {}
for item in config_by_roles.items:
role_group_name = item.name
_.set(role_config_group_name_displayname_mapping, "" + item.display_name,
role_group_name)
for subitem in item.config.items:
_.set(
