def set_secret(keyvault_name: str, secret_name: str, value: str): if value: az_cli("keyvault secret set", "--name", secret_name, "--vault-name", keyvault_name, "--value", value) else: print("WARN: Provided secret '%s' value is empty, ignoring request " % secret_name)
def prompt_or_create_ad_group(msg: str, tenant_id='default', provided_ad_group_id: str = None, create_if_not_exists=False, no_input: bool = False, add_signed_user: bool = False): """ Prompt for existing or create a new AD group :param msg: :param tenant_id: :param create_if_not_exists: :param add_signed_user: :return: """ group_accepted = False ad_group = provided_ad_group_id while not group_accepted: if not ad_group: ad_group = input(msg) print('\n') if is_valid_uuid(ad_group): rsp = az_cli("ad group list ", "--filter", "objectId eq '%s' " % ad_group) else: rsp = az_cli("ad group list ", "--filter", "displayname eq '%s' " % ad_group) if len(rsp) > 1: print("There are multiple groups found in your %s tenant " % tenant_id) if create_if_not_exists: print("Please select unique name/objectId or enter new one to create: ") else: print("Please select unique name/objectId : ") print("ObjectId\tName ") for group in rsp: print("%s\t%s" % (group['objectId'], group['displayName'])) elif len(rsp) == 1: group = rsp[0] if not no_input: group_accepted = yes_no("Selected group: %s, objectId: %s, confirm (y/n)" % (group['displayName'], group['objectId'])) # reset selection if not accepted to prompt again ad_group = ad_group if group_accepted else None else: # no confirmation needed for non-interactive mode group_accepted = True else: if create_if_not_exists: desc = "AD group for Watercooler app administrators. It manages full access to App resources and SQL database" az_cli("ad group create --display-name %s --mail-nickname %s " % ( ad_group, ad_group.lower()), "--description", desc ) group_accepted = True group = az_cli("ad group show ", "--group", ad_group) if add_signed_user: _add_signed_user_to_group(existing_group_name=ad_group) return { "ad_group_name": group['displayName'], "objectId": group['objectId'] }
def enable_webapp_alert(resource_group: str, alert_name: str, enabled: bool = True): try: # This call will fail if alert does not exist az_cli("monitor metrics alert show", "--resource-group", resource_group, "--name", alert_name) return az_cli("monitor metrics alert update", "--resource-group", resource_group, "--name", alert_name, "--enabled", str(enabled).lower()) except BaseException as er: return None
def add_role_assigment(role: str, ad_group_id: str, resource_group: str): """ Adds role assigment for ad_group_id on resourec group level :param role: :param ad_group_id: object ids for users, groups, service principals, and managed identities. :return: """ rsp = az_cli("role assignment create", "--role", role, "--assignee-object-id", ad_group_id, "--resource-group", resource_group) return rsp
def _get_access_token(resource_id: str = None, subscription_id: str = None): rsp = dict() az_cli_args = [] if resource_id: az_cli_args.extend(["--resource", resource_id]) if subscription_id: az_cli_args.extend(["--subscription", subscription_id]) rsp = az_cli(" account get-access-token ", *az_cli_args) print("The access token for resource %s expires at %s" % (resource_id, rsp.get('expiresOn'))) return rsp.get('accessToken')
def get_loggedin_user(fields: list): query = "" if fields: query = "{" + ",".join(list(map(lambda s: s + " : " + s, fields))) + " }" try: rsp = az_cli("ad signed-in-user show", "--query", query) return rsp except BaseException: print("WARN: Can't resolve signed in user, are you running as service principal?") return None
def initialize_databricks_cluster(install_config: InstallConfiguration, resource_group: str, artifacts_path: str, tenant_id: str = None, subscription_id: str = None): runtime_storage = install_config.runtime_storage_account_name storage_account_keys_list_res = az.az_cli( "storage account keys list --account-name " + runtime_storage) storage_account_access_key = storage_account_keys_list_res[0]["value"] print("Creating Databricks cluster ... ") backend_keyvault_name = install_config.backend_keyvault_name wc_sp_secret_value = install_config.wc_service_principal['password'] adb_ws_name = install_config.databricks_workspace_name ws_url = arm_ops.get_databricks_workspace_url( resource_group=resource_group, ws_name=adb_ws_name) if not ws_url.startswith("https://"): ws_url = "https://" + ws_url adb_access_token = ad_ops.get_databricks_access_token( tenant_id, subscription_id) managed_libraries = [] with open("cluster_libraries.json", "r") as libs_file: managed_libraries = json.load(libs_file) provisioning_rsp = arm_ops.provision_databricks_cluster( install_config=install_config, workspace_url=ws_url, oauth_access_token=adb_access_token, wc_sp_secret_value=wc_sp_secret_value, managed_libraries=managed_libraries) install_config.adb_cluster_details = provisioning_rsp upload_mount_storage_file(ws_url, adb_access_token) execute_script_mount_storage_script( databricks_host=ws_url, token=adb_access_token, cluster_id=provisioning_rsp['cluster_id'], storage_account_name=runtime_storage, container_name="data", secret_key=storage_account_access_key) secrets_ops.set_secret(keyvault_name=backend_keyvault_name, secret_name="wc-databricks-token", value=provisioning_rsp['api_token']['token_value']) print("Uploading artifacts onto DBFS ...") arm_ops.upload_artifacts(workspace_url=ws_url, oauth_access_token=adb_access_token, local_artifacts_path=artifacts_path, dbfs_dir_path="/mnt/watercooler/scripts")
def _find_build_in_role(aad_app_id: str, role_name: str = None, permission_name: str = None): if not role_name and not permission_name: raise ValueError("role_name or permission_name must be provided") if role_name and permission_name: raise ValueError("role_name or permission_name are mutual exclusive") app = az_cli("ad sp show", "--id", aad_app_id, "--query", "{Name:appDisplayName, Id:appId}") if app: app_id = app['Id'] query = "appRoles[?value=='%s'].{Value:value, Id:id}" % role_name if permission_name: query = "oauth2Permissions[?value=='%s'].{Value:value, Id:id}" % permission_name roles_rsp = az_cli("ad sp show", "--id", app_id, "--query", query) if len(roles_rsp) == 1: role_id = roles_rsp[0]['Id'] return { "appId": app_id, "id": role_id } else: print("Cant' find unique result for '%s' in '%s'" % (role_name, aad_app_id)) return None
def get_watercooler_user(): while True: mail = input("Please enter valid watercooler mail for calendar events mail sender account:") print("The following mail was introduced:"+str(mail)) correct = yes_no("Is this correct?") if not correct: continue existing_users = az_cli("ad user list ", "--filter", "mail eq '%s' " % mail) if len(existing_users)==0: print("Given user was not found") continue else: user_json_info = existing_users[0] return user_json_info
def copy_file(source_path: str, resource_group: str, runtime_storage: str, dest_container_name: str, dest_path): storage_connection_str = az.az_cli( "storage account show-connection-string", "--resource-group", resource_group, "--name", runtime_storage, "--query", "connectionString") dest_service = BlockBlobService(account_name=runtime_storage, connection_string=storage_connection_str) print("Copying %s into %s : %s/%s " % (source_path, runtime_storage, dest_container_name, dest_path)) dest_service.create_blob_from_path(container_name=dest_container_name, blob_name=dest_path, file_path=source_path)
def add_wc_deployer_identity(resource_group: str, wc_admin_ad_group: str, identity_name: str = "wc-deployer"): group_id = az_cli("group show", "--name", resource_group, "--query", "id") id_principals = az_cli("identity list", "--resource-group", resource_group, "--query", "[?name=='%s'].principalId" % identity_name) identity = dict() if len(id_principals) == 0: print("Creating user-assigned identity %s in resource group %s " % (identity_name, resource_group)) identity = az_cli("identity create", "--resource-group", resource_group, "--name", identity_name) sleep(5) rsp = az_cli("identity list", "--resource-group", resource_group, "--query", "[?principalId=='%s']" % identity['principalId']) if not rsp: print("Warning: couldn't find wc-deployer identity ") # azure is slow.. let's wait more sleep(5) else: print("Identity %s exists already in group %s " % (identity_name, resource_group)) identity = az_cli("identity show", "--resource-group", resource_group, "--name", identity_name) _add_member_to_group(existing_group_name=wc_admin_ad_group, user_or_identity_obj_id=identity['principalId']) az_cli("role assignment create", "--role", "Contributor", "--assignee-object-id", identity['principalId'], "--scope", group_id) return identity
def get_service_principal_object_id(app_id: str): rsp = az_cli("ad sp show", "--id", app_id, "--query", "objectId") return rsp
def get_log_workspace_key(resource_group: str, workspace_name: str): return az_cli("monitor log-analytics workspace get-shared-keys", "--resource-group", resource_group, "--workspace-name", workspace_name, "--query", "primarySharedKey")
def find_log_workspace_id(resource_group: str, workspace_name: str): return az_cli("monitor log-analytics workspace show", "--resource-group", resource_group, "--workspace-name", workspace_name, "--query", "customerId")
def restart_web_app(resource_group: str, app_name: str): az_cli("webapp restart", "--resource-group", resource_group, "--name", app_name)
def make_user_owner_for_app(user_object_id: str, app_id: str): return az_cli("ad app owner add --id", app_id, "--owner-object-id", user_object_id)
def add_service_principal_delegated_permission(sp_app_id: str, api_resource_id: str, permission_id: str): rsp = az_cli("ad app permission add", "--id", sp_app_id, "--api", api_resource_id, "--api-permissions", "%s=Scope" % permission_id) print(rsp)
arm_params_json_file = os.path.join(install_config.get_wc_dir(), "wc_arm_params.json") with open(arm_params_json_file, "w") as param_file: param_file.write(json_params) print("Saved deployment parameters at %s " % arm_params_json_file) print("Validating ARM template with given parameters") arm_ops.validate_templates(template_uri=main_template_uri, param_file_path=arm_params_json_file, resource_group=rs_group_name) print("Deploying ARM template mainTemplate.json") arm_ops.deploy_arm_template(template_uri=main_template_uri, param_file_path=arm_params_json_file, resource_group=rs_group_name) if __name__ == '__main__': args = sys.argv current_account = az.az_cli("account show") # Create the parser arg_parser = argparse.ArgumentParser(description='Install Watercooler service') # Add the arguments arg_parser.add_argument('--deployment-name', metavar='deployment-name', type=str, help='Name of deployment', required=True) arg_parser.add_argument('--tenant-id', metavar='tenant-id', type=str, help='Id of Azure tenant used for deployment', required=True) arg_parser.add_argument('--subscription-id', metavar='subscription-id',
def _add_member_to_group(existing_group_name: str, user_or_identity_obj_id: str): is_member = az_cli("ad group member check", "--member-id", user_or_identity_obj_id, "--group", existing_group_name, "--query", "value") if not is_member: az_cli("ad group member add", "--member-id", user_or_identity_obj_id, "--group", existing_group_name)
def get_or_create_service_principal(name: str, tenant_id: str, non_interactive_mode: bool, create_if_not_exists: bool = True, credentials_valid_years: int = 1, is_web_app: bool = False, **kwargs): if is_web_app: assert 'reply_url' in kwargs assert 'logout_url' in kwargs if not non_interactive_mode: service_principal = dict() sp_accepted = False while not sp_accepted: existing_sps = az_cli("ad sp list ", "--all", "--filter", "displayName eq '%s' " % name, "--query", "[?contains(appOwnerTenantId, '%s')]" % tenant_id) if len(existing_sps) > 1: print("There are multiple service principal found in your tenant ") if create_if_not_exists: print("Please select unique name/objectId or enter new one to create: ") else: print("Please select unique name/objectId : ") print("appId\tName ") for sp in existing_sps: print("%s\t%s" % (sp['appId'], sp['displayName'])) app_id = input("If you want to use one of the service principals enter one of the app ids, otherwise press enter: ") selected_sp = None if app_id: for sp in existing_sps: if sp['appId'] == app_id: selected_sp = sp break else: name = input("Enter a name for a new service principal: ") if selected_sp: password = getpass.getpass(prompt="Service Principal Secret") conf_password = getpass.getpass(prompt="Confirm Service Principal Secret") service_principal = { "appId": sp['appId'], "password": password, "name": name } sp_accepted = password != "" and password == conf_password elif len(existing_sps) == 1: sp = existing_sps[0] print("\nFound an existing service principal : %s, appId: %s. " % (sp['displayName'], sp['appId'])) print("The service principal with this name is required for the deployment process. You can either reuse this service principal or abort the deployment.") print("Reusing it requires you to provide a service principal client secret. ") print("If you are an owner of the service principal, you also have the option of creating a new secret and providing that (e.g. if you don't know the current one).") sp_accepted = yes_no("Would you like to reuse the existing service principal? (y/n)") print("\n") if sp_accepted: password = getpass.getpass(prompt="Service Principal Secret") conf_password = getpass.getpass(prompt="Confirm Service Principal Secret") service_principal = { "appId": sp['appId'], "password": password, "name": name } sp_accepted = password != "" and password == conf_password else: name = input("Enter a name different from %s, that will be used to create a new service principal: " % name) else: if create_if_not_exists: if is_web_app: service_principal = _create_web_app_service_principal(display_name=name, reply_url=kwargs['reply_url'], logout_url=kwargs['logout_url'], credentials_valid_years=credentials_valid_years) sp_accepted = True else: sp = az_cli("ad sp create-for-rbac", "--name", name, "--years", str(credentials_valid_years)) service_principal = { "appId": sp['appId'], "password": sp['password'], "name": name } sp_accepted = True else: return None else: # if the application is running in non interactive mode even if there is a service principal # with the given name we will create a new one if is_web_app: service_principal = _create_web_app_service_principal(display_name=name, reply_url=kwargs['reply_url'], logout_url=kwargs['logout_url'], credentials_valid_years=credentials_valid_years) else: sp = az_cli("ad sp create-for-rbac", "--name", name, "--years", str(credentials_valid_years)) service_principal = { "appId": sp['appId'], "password": sp['password'], "name": name } return service_principal
def _add_signed_user_to_group(existing_group_name: str): signed_user = az_cli("ad signed-in-user show", "--query", "objectId") _add_member_to_group(existing_group_name, signed_user)
def update_linked_service(resource_group: str, factory_name: str, service_name: str, prop_name: str, value: str): az_cli("datafactory linked-service update", "--resource-group", resource_group, "--factory-name", factory_name, "--linked-service-name", service_name, "--set", "%s=%s" % (prop_name, value))
def get_databricks_workspace_id(resource_group: str, ws_name: str): return az_cli("databricks workspace show", "--name", ws_name, "--resource-group", resource_group, "--query", "id")
def _create_web_app_service_principal(display_name, reply_url, logout_url, credentials_valid_years: int = 1): graph_permission = find_graph_user_read_permission() azure_service_management_permission = find_azure_service_management_user_impersonation_permission() azure_storage_permission = find_azure_storage_user_impersonation_permission() if not graph_permission: raise RuntimeError("Couldn't find 'User.Read' permission reference") if not azure_service_management_permission: raise RuntimeError("Couldn't find 'user_impersonation' permission reference") tmp_file_name = "/tmp/" + str(uuid.uuid4()) + ".json" with open(tmp_file_name, mode="w") as fp: graphAppId = graph_permission['appId'] graph_permission_id = graph_permission['id'] azure_service_management_app_id = azure_service_management_permission['appId'] azure_service_management_permission_id = azure_service_management_permission['id'] azure_storage_app_id = azure_storage_permission['appId'] azure_storage_permission_id = azure_storage_permission['id'] permissions = [{ "resourceAppId": graphAppId, "resourceAccess": [ { "id": graph_permission_id, "type": "Scope" } ] }, { "resourceAppId": azure_service_management_app_id, "resourceAccess": [ { "id": azure_service_management_permission_id, "type": "Scope" } ] }, { "resourceAppId": azure_storage_app_id, "resourceAccess": [ { "id": azure_storage_permission_id, "type": "Scope" } ] }] json.dump(permissions, fp) fp.flush() try: password = make_strong_password(42, '/+') start_date = date.today() end_date = date(start_date.year + credentials_valid_years, start_date.month, start_date.day) rsp = az_cli("ad app create", "--available-to-other-tenants", "false", "--password", password, "--display-name", display_name, "--end-date", str(end_date), "--reply-urls", reply_url, "--required-resource-accesses", "@%s" % tmp_file_name) _update_logout_url(logout_url, app_id=rsp['appId']) _update_group_membership_claims('SecurityGroup', app_id=rsp['appId']) service_principal = { "appId": rsp['appId'], "password": password, "name": display_name } return service_principal finally: if os.path.exists(tmp_file_name): try: os.remove(tmp_file_name) except BaseException as e: pass
def _update_group_membership_claims(groupMembershipClaims: str, app_id: str): az_cli("ad app update", "--id", app_id, "--set", "groupMembershipClaims=\"%s\"" % groupMembershipClaims)
def _update_logout_url(logout_url: str, app_id: str): az_cli("ad app update", "--id", app_id, "--set", "logoutUrl=\"%s\"" % logout_url)
def activate_datafactory_trigger(resource_group: str, factory_name: str, trigger_name: str): az_cli("datafactory trigger start", "--factory-name", factory_name, "--resource-group", resource_group, "--name", trigger_name)
def get_group_members(group_object_id: str): return az_cli("ad group member list --group", group_object_id)
def deploy_arm_template(template_uri: str, param_file_path: str, resource_group: str): return az_cli("deployment group create", "--resource-group", resource_group, "--template-uri", template_uri, "--parameters", param_file_path)
def _copy_data(resource_group: str, runtime_storage: str, source_subfolder: str, dest_container_name, source_storage_account_name: str, source_container_name: str, destination_path_prefix: str = None): source_container_url: str = f"https://{source_storage_account_name}.blob.core.windows.net/{source_container_name}" files = list_blobs_in(container_url=source_container_url, subfolder=source_subfolder) storage_connection_str = az.az_cli( "storage account show-connection-string", "--resource-group", resource_group, "--name", runtime_storage, "--query", "connectionString") source_block_blob_service = BlockBlobService( account_name=source_storage_account_name) dest_service = BlockBlobService(account_name=runtime_storage, connection_string=storage_connection_str) for blob_path in files: source_url = source_container_url + "/" + blob_path target_file_path = blob_path[len(source_subfolder) + 1:] if blob_path.startswith( source_subfolder + "/") else blob_path if destination_path_prefix: # make sure there is no leading slash otherwise blob path is going to be broken with double slashes target_file_path = destination_path_prefix.lstrip( "/") + target_file_path print("Copying %s into %s/%s " % (blob_path, dest_container_name, target_file_path)) # We must wait until copying completely finished so that the next deployment steps have all the required data, # however, setting requires_sync=True does not support blobs larger than 256MB. # Therefore, we need to use async copy together with copied blob properties polling dest_service.copy_blob(container_name=dest_container_name, blob_name=target_file_path, copy_source=source_url) copy_finished = False while not copy_finished: time.sleep(10) dest_blob_properties = dest_service.get_blob_properties( container_name=dest_container_name, blob_name=target_file_path).properties source_blob_properties = source_block_blob_service.get_blob_properties( container_name=source_container_name, blob_name=blob_path).properties copy_finished = dest_blob_properties.content_settings.content_md5 == source_blob_properties.content_settings.content_md5 \ and dest_blob_properties.content_length == source_blob_properties.content_length if not copy_finished: print( "Waiting for copy operation to finish. Source content_md5=", source_blob_properties.content_settings.content_md5, " destination content_md5=", dest_blob_properties.content_settings.content_md5, " source content_length=", source_blob_properties.content_length, " destination content_length=", dest_blob_properties.content_length) else: print( "Copy operation finished successfully. Source content_md5=", source_blob_properties.content_settings.content_md5, " destination content_md5=", dest_blob_properties.content_settings.content_md5, " source content_length=", source_blob_properties.content_length, " destination content_length=", dest_blob_properties.content_length)