def prepare_user(username, password):
"""
Create a new user account (admin) for Cloudera Director Server
:param username: Username for the new account
:param password: Password for the new account
:return: API exit code
"""
# Cloudera Director server runs at http://127.0.0.1:7189
try:
logging.info('Creating new admin user for Cloudera Director Server')
client = ApiClient("http://localhost:7189")
AuthenticationApi(client).login(
Login(username="******",
password="******")) # create new login base on user input
users_api = UsersApi(client)
# Admin user by default has both roles
users_api.create(
User(username=username,
password=password,
enabled=True,
roles=["ROLE_ADMIN", "ROLE_READONLY"]))
logging.info('Successfully created new admin user %s.' % dirUsername)
except HTTPError, e:
logging.error("Failed to create user '%s'. %s" % (username, e.msg))
return ExitCodes.ERROR
def _verify_azure_update_environment(ip, password):
address = "http://%s:7189" % ip
print(address)
client = ApiClient(address)
AuthenticationApi(client).login(Login(username="******", password=password))
environments_api = EnvironmentsApi(client)
list_env = environments_api.list()
print("env list ->" + str(list_env))
if list_env:
for environments in list_env:
environment_information = environments_api.getRedacted(environments)
if environment_information.provider.type == 'azure':
print("Azure Environment: " + environments)
b = {"subscriptionId":
environment_information.provider.config['subscriptionId'],
"tenantId":
environment_information.provider.config['tenantId']
}
c = {
"clientId": CLIENT_ID,
"clientSecret": CLIENT_SECRET
}
d = dict(b,**c)
assert CLIENT_ID == \
environment_information.provider.config['clientId']
print(json.dumps(d))
environments_api.updateProviderCredentials(environments, (d))
def get_authenticated_client(args):
"""
Create a new API client and authenticate against a server as admin
@param args: dict of parsed command line arguments that
include server host and admin credentials
@rtype: ApiClient
@return: authenticated API client
"""
# Start by creating a client pointing to the right server
client = ApiClient(args.server)
# Authenticate. This will start a session and store the cookie
auth = AuthenticationApi(client)
auth.login(Login(username=args.admin_username, password=args.admin_password))
return client
def get_authenticated_client(self):
"""
Create a new API client and authenticate against a server as admin
@param server: director server url
@param admin_username: user with administrative access
@param admin_password: password for admin user
"""
# Start by creating a client pointing to the right server
client = ApiClient(self.server)
# Authenticate. This will start a session and store the cookie
auth = AuthenticationApi(client)
auth.login(
Login(username=self.admin_username, password=self.admin_password))
self.client = client
def get_authenticated_client(args):
"""
Create a new API client and authenticate against a server as admin
@param args: dict of parsed command line arguments that
include server host and admin credentials
@rtype: ApiClient
@return: authenticated API client
"""
# Start by creating a client pointing to the right server
client = ApiClient(args.server)
# Authenticate. This will start a session and store the cookie
auth = AuthenticationApi(client)
auth.login(Login(username=args.admin_username, password=args.admin_password))
return client
def main(arguments):
# Get all command line arguments
cloudera_director_server = arguments[0]
admin_username = arguments[1]
credentials_file_path = arguments[2]
admin_password = open(credentials_file_path, 'r').read()
num_lookback_dates = arguments[3]
# Optional arguments for transient clusters
cluster_name = ''
if ((len(arguments)) > 4):
cluster_name = arguments[4]
# Setup a Cloudera Director Client
client = ApiClient(cloudera_director_server)
AuthenticationApi(client).login(
Login(username=admin_username, password=admin_password))
# Get all Environments
environments = EnvironmentsApi(client).list()
if not environments:
sys.exit(1)
# Get start and end time of the query
local_tz = timezone('US/Eastern')
from_time = datetime.now() - timedelta(hours=8)
from_time = from_time.replace(tzinfo=local_tz)
to_time = datetime.now().replace(tzinfo=local_tz)
# Iterate through all environments to get all deployments
for environment in environments:
deployments = DeploymentsApi(client).list(environment)
if not deployments:
continue
# Iterate through all deployments to get all clusters
for deployment in deployments:
clusters = ClustersApi(client).list(environment, deployment)
if not clusters:
continue
# Iterate through all clusters to run queries
for cluster in clusters:
#Filter only the cluster if cluster name passed as argument
if (cluster_name != '' and cluster_name != cluster):
continue
print(
"Get the usage of cluster [%s] in deployment [%s] in environment [%s] from [%s] to [%s] "
% (cluster, deployment, environment, from_time, to_time))
runQuery(client, environment, deployment, cluster, from_time,
to_time)
def main():
parser = argparse.ArgumentParser(prog='ephemeral-spark-submit.py')
parser.add_argument(
'--admin-username',
default="admin",
help=
'Name of an user with administrative access (defaults to %(default)s)')
parser.add_argument(
'--admin-password',
default="admin",
help='Password for the administrative user (defaults to %(default)s)')
parser.add_argument(
'--server',
default="http://localhost:7189",
help="Cloudera Director server URL (defaults to %(default)s)")
parser.add_argument(
'--cm',
help="The name of the Cloudera Manager server to use in Director")
parser.add_argument('--environment',
help="The name of the Environment to use in Director")
parser.add_argument(
'--jar', help="JAR for Spark job you want to run on ephemeral cluster")
parser.add_argument('--jarclass', help="The --class flag for spark-submit")
parser.add_argument('--args', help="The arguments for the jar")
parser.add_argument('--script', help="Script that runs before spark job")
parser.add_argument('config_file',
help="Cluster configuration file (.ini)")
args = parser.parse_args()
if not isfile(args.config_file):
print 'Error: "%s" not found or not a file' % args.config_file
return -1
config = ConfigParser.SafeConfigParser()
config.read(args.config_file)
#Create authenticated client
client = cluster.get_authenticated_client(args)
#Execute cluster creation
cluster_name = cluster.create_cluster(client, args.environment, args.cm,
config)
print 'Waiting for the cluster to be ready. Check the web interface for details.'
cluster.wait_for_cluster(client, args.environment, args.cm, cluster_name)
client = ApiClient(args.server)
AuthenticationApi(client).login(
Login(username=args.admin_username, password=args.admin_password))
clusters = ClustersApi(client)
eph_cluster = clusters.get(args.environment, args.cm, cluster_name)
instances = eph_cluster.instances
#Find which is a gateway node
for instance in instances:
if str(instance.virtualInstance.template.name) == 'gateway':
gateway = instance
gateway = gateway.properties['publicDnsName']
print("The Gateway url is: " + gateway)
#Copy the JAR and postscript to the GW
copy_jar(args.jar, gateway, config)
#Copy script to the GW
copy_script(args.script, gateway, config)
#Create directory in HDFS with correct permissions
configure_hdfs(gateway, config)
#Execute the job
execute_spark(args.jar, args.jarclass, args.args, gateway, config)
#Run some post script
execute_script(args.script, gateway, config)
#Destroy the cluster
print "Job complete, terminating the instance"
clusters.delete(args.environment, args.cm, cluster_name)
return 0
User(username=username,
password=password,
enabled=True,
roles=["ROLE_ADMIN", "ROLE_READONLY"]))
logging.info('Successfully created new admin user %s.' % dirUsername)
except HTTPError, e:
logging.error("Failed to create user '%s'. %s" % (username, e.msg))
return ExitCodes.ERROR
# delete existing admin user using the new account
try:
logging.info(
"Deleting default user 'admin' for Cloudera Director Server")
client = ApiClient("http://localhost:7189")
AuthenticationApi(client).login(
Login(username=username, password=password))
users_api = UsersApi(client)
users_api.delete("admin")
logging.info("Successfully deleted default user 'admin'")
return ExitCodes.OK
except HTTPError, e:
logging.error("Failed to delete default user 'admin'. %s" % e.msg)
return ExitCodes.ERROR
dirUsername = sys.argv[1]
dirPassword = sys.argv[2]
sys.exit(prepare_user(dirUsername, dirPassword))