def prepare_user(username, password):
"""
Create a new user account (admin) for Cloudera Director Server
:param username: Username for the new account
:param password: Password for the new account
:return: API exit code
"""
# Cloudera Director server runs at http://127.0.0.1:7189
try:
logging.info('Creating new admin user for Cloudera Director Server')
client = ApiClient("http://localhost:7189")
AuthenticationApi(client).login(
Login(username="******",
password="******")) # create new login base on user input
users_api = UsersApi(client)
# Admin user by default has both roles
users_api.create(
User(username=username,
password=password,
enabled=True,
roles=["ROLE_ADMIN", "ROLE_READONLY"]))
logging.info('Successfully created new admin user %s.' % dirUsername)
except HTTPError, e:
logging.error("Failed to create user '%s'. %s" % (username, e.msg))
return ExitCodes.ERROR
def _verify_azure_update_environment(ip, password):
address = "http://%s:7189" % ip
print(address)
client = ApiClient(address)
AuthenticationApi(client).login(Login(username="******", password=password))
environments_api = EnvironmentsApi(client)
list_env = environments_api.list()
print("env list ->" + str(list_env))
if list_env:
for environments in list_env:
environment_information = environments_api.getRedacted(environments)
if environment_information.provider.type == 'azure':
print("Azure Environment: " + environments)
b = {"subscriptionId":
environment_information.provider.config['subscriptionId'],
"tenantId":
environment_information.provider.config['tenantId']
}
c = {
"clientId": CLIENT_ID,
"clientSecret": CLIENT_SECRET
}
d = dict(b,**c)
assert CLIENT_ID == \
environment_information.provider.config['clientId']
print(json.dumps(d))
environments_api.updateProviderCredentials(environments, (d))
def main(arguments):
# Get all command line arguments
cloudera_director_server = arguments[0]
admin_username = arguments[1]
credentials_file_path = arguments[2]
admin_password = open(credentials_file_path, 'r').read()
num_lookback_dates = arguments[3]
# Optional arguments for transient clusters
cluster_name = ''
if ((len(arguments)) > 4):
cluster_name = arguments[4]
# Setup a Cloudera Director Client
client = ApiClient(cloudera_director_server)
AuthenticationApi(client).login(
Login(username=admin_username, password=admin_password))
# Get all Environments
environments = EnvironmentsApi(client).list()
if not environments:
sys.exit(1)
# Get start and end time of the query
local_tz = timezone('US/Eastern')
from_time = datetime.now() - timedelta(hours=8)
from_time = from_time.replace(tzinfo=local_tz)
to_time = datetime.now().replace(tzinfo=local_tz)
# Iterate through all environments to get all deployments
for environment in environments:
deployments = DeploymentsApi(client).list(environment)
if not deployments:
continue
# Iterate through all deployments to get all clusters
for deployment in deployments:
clusters = ClustersApi(client).list(environment, deployment)
if not clusters:
continue
# Iterate through all clusters to run queries
for cluster in clusters:
#Filter only the cluster if cluster name passed as argument
if (cluster_name != '' and cluster_name != cluster):
continue
print(
"Get the usage of cluster [%s] in deployment [%s] in environment [%s] from [%s] to [%s] "
% (cluster, deployment, environment, from_time, to_time))
runQuery(client, environment, deployment, cluster, from_time,
to_time)
def get_authenticated_client(args):
"""
Create a new API client and authenticate against a server as admin
@param args: dict of parsed command line arguments that
include server host and admin credentials
@rtype: ApiClient
@return: authenticated API client
"""
configuration = Configuration()
configuration.host = args.server
configuration.username = args.admin_username
configuration.password = args.admin_password
configuration.ssl_ca_cert = args.cafile
return ApiClient(configuration=configuration)
def get_authenticated_client(self):
"""
Create a new API client and authenticate against a server as admin
@param server: director server url
@param admin_username: user with administrative access
@param admin_password: password for admin user
"""
# Start by creating a client pointing to the right server
client = ApiClient(self.server)
# Authenticate. This will start a session and store the cookie
auth = AuthenticationApi(client)
auth.login(
Login(username=self.admin_username, password=self.admin_password))
self.client = client
def get_authenticated_client(args):
"""
Create a new API client and authenticate against a server as admin
@param args: dict of parsed command line arguments that
include server host and admin credentials
@rtype: ApiClient
@return: authenticated API client
"""
# Start by creating a client pointing to the right server
client = ApiClient(args.server)
# Authenticate. This will start a session and store the cookie
auth = AuthenticationApi(client)
auth.login(Login(username=args.admin_username, password=args.admin_password))
return client
def __init__(self, api_client=None):
if api_client is None:
api_client = ApiClient()
self.api_client = api_client
def service_login(service='cloudbreak',
username=None,
password=None,
bool_response=False):
"""
Login to the currently configured server.
Login requires a secure connection over https.
Prior to calling this method, the host must be specified
and the SSLContext should be configured (if necessary).
Successful login will result in a generated token (JWT) being
cached in the api_client config that will be passed in all future
REST API calls. To clear that token, call service_logout.
The token is temporary and will expire after a duration set by
the server. After a token expires, you must call
this method again to generate a new token.
Args:
service (str): 'cloudbreak'; the service to login to
username (str): The username to submit
password (str): The password to use
bool_response (bool): If True, the function will return False instead
of an error. Useful for connection testing.
Returns:
(bool): True if successful, False or an Error if not. See bool_response
"""
log_args = locals()
horton = utils.Horton()
_ = log_args.pop('password')
log.info("Called service_login with args %s", log_args)
assert service in _valid_services
assert isinstance(username, six.string_types)
assert isinstance(password, six.string_types)
assert isinstance(bool_response, bool)
if service == 'cloudbreak':
configuration = config.cb_config
elif service == 'director':
configuration = config.cd_config
else:
raise ValueError("Unrecognised Service parameter")
assert configuration.host, "Host must be set prior to logging in."
assert configuration.host.startswith("https"), \
"Login is only available when connecting over HTTPS."
if service == 'director':
config.cd_config.username = username
config.cd_config.password = password
horton.cad = ApiClient(configuration=config.cd_config)
return True
if service == 'cloudbreak':
url = config.cb_config.host.replace('/cb/api',
'/identity/oauth/authorize')
redirect_uri = config.cb_config.host.replace('/cb/api', '/authorize')
resp = requests.post(
url=url,
params={
'response_type': 'token',
'client_id': 'cloudbreak_shell',
'scope.0': 'openid',
'source': 'login',
'redirect_uri': 'http://cloudbreak.shell'
},
headers={'accept': 'application/x-www-form-urlencoded'},
verify=config.cb_config.verify_ssl,
allow_redirects=False,
data=[
('credentials', '{"username":"******",'
'"password":"******"}'),
])
if 'Location' not in resp.headers:
raise ValueError("Login Failed, please check Cloudbreak and your"
"Credentials")
try:
token = parse.parse_qs(resp.headers['Location'])['access_token'][0]
# Todo: get the expiry and set into config as well
# Todo: use expiry to refresh the token as required
# Todo: Find approach to auto fetch the token as required
except KeyError:
if bool_response:
return False
raise ConnectionError("No Access Token in server response. Please "
"check your Auth config and environment.")
set_service_auth_token(token=token, service='cloudbreak')
return True
def main():
parser = argparse.ArgumentParser(prog='ephemeral-spark-submit.py')
parser.add_argument(
'--admin-username',
default="admin",
help=
'Name of an user with administrative access (defaults to %(default)s)')
parser.add_argument(
'--admin-password',
default="admin",
help='Password for the administrative user (defaults to %(default)s)')
parser.add_argument(
'--server',
default="http://localhost:7189",
help="Cloudera Director server URL (defaults to %(default)s)")
parser.add_argument(
'--cm',
help="The name of the Cloudera Manager server to use in Director")
parser.add_argument('--environment',
help="The name of the Environment to use in Director")
parser.add_argument(
'--jar', help="JAR for Spark job you want to run on ephemeral cluster")
parser.add_argument('--jarclass', help="The --class flag for spark-submit")
parser.add_argument('--args', help="The arguments for the jar")
parser.add_argument('--script', help="Script that runs before spark job")
parser.add_argument('config_file',
help="Cluster configuration file (.ini)")
args = parser.parse_args()
if not isfile(args.config_file):
print 'Error: "%s" not found or not a file' % args.config_file
return -1
config = ConfigParser.SafeConfigParser()
config.read(args.config_file)
#Create authenticated client
client = cluster.get_authenticated_client(args)
#Execute cluster creation
cluster_name = cluster.create_cluster(client, args.environment, args.cm,
config)
print 'Waiting for the cluster to be ready. Check the web interface for details.'
cluster.wait_for_cluster(client, args.environment, args.cm, cluster_name)
client = ApiClient(args.server)
AuthenticationApi(client).login(
Login(username=args.admin_username, password=args.admin_password))
clusters = ClustersApi(client)
eph_cluster = clusters.get(args.environment, args.cm, cluster_name)
instances = eph_cluster.instances
#Find which is a gateway node
for instance in instances:
if str(instance.virtualInstance.template.name) == 'gateway':
gateway = instance
gateway = gateway.properties['publicDnsName']
print("The Gateway url is: " + gateway)
#Copy the JAR and postscript to the GW
copy_jar(args.jar, gateway, config)
#Copy script to the GW
copy_script(args.script, gateway, config)
#Create directory in HDFS with correct permissions
configure_hdfs(gateway, config)
#Execute the job
execute_spark(args.jar, args.jarclass, args.args, gateway, config)
#Run some post script
execute_script(args.script, gateway, config)
#Destroy the cluster
print "Job complete, terminating the instance"
clusters.delete(args.environment, args.cm, cluster_name)
return 0
users_api.create(
User(username=username,
password=password,
enabled=True,
roles=["ROLE_ADMIN", "ROLE_READONLY"]))
logging.info('Successfully created new admin user %s.' % dirUsername)
except HTTPError, e:
logging.error("Failed to create user '%s'. %s" % (username, e.msg))
return ExitCodes.ERROR
# delete existing admin user using the new account
try:
logging.info(
"Deleting default user 'admin' for Cloudera Director Server")
client = ApiClient("http://localhost:7189")
AuthenticationApi(client).login(
Login(username=username, password=password))
users_api = UsersApi(client)
users_api.delete("admin")
logging.info("Successfully deleted default user 'admin'")
return ExitCodes.OK
except HTTPError, e:
logging.error("Failed to delete default user 'admin'. %s" % e.msg)
return ExitCodes.ERROR
dirUsername = sys.argv[1]
dirPassword = sys.argv[2]
