def test_validate_json_valid_inputs(): """ Unit test to check the validate_json function with valid inputs """ json_object = { "name": "workspace-name", "resource_group": "resource-group-name", "create_workspace": False, "friendly_name": "friendly-name", "create_resource_group": True, "location": "location", "sku": "basic", "storage_account": "Microsoft.Storage/storageAccounts/<your-storage-account-name>", "key_vault": "Microsoft.KeyVault/vaults/<your-key-vault-name>", "app_insights": "Microsoft.Insights/components/<your-app-insights-name>", "container_registry": "Microsoft.ContainerRegistry/registries/<your-container-registry-name>", "cmk_key_vault": "Microsoft.KeyVault/vaults/<your-key-vault-name>", "resource_cmk_uri": "https://<your-cmk-uri>", "hbi_workspace": False } validate_json(data=json_object, schema=parameters_schema, input_name="PARAMETERS_FILE")
def test_validate_json_valid_inputs(): """ Unit test to check the validate_json function with valid inputs """ json_object = { "model_name": "model-name", "model_file_name": "model-file-name", "model_framework": "onnx", "metrics_min": [] } validate_json(data=json_object, schema=parameters_schema, input_name="PARAMETERS_FILE")
def check_if_agents_exist(context, orb_tags, status): context.agent_name = generate_random_string_with_predefined_prefix( agent_name_prefix) context.orb_tags = create_tags_set(orb_tags) context.agent = create_agent(context.token, context.agent_name, context.orb_tags) context.agent_key = context.agent["key"] token = context.token run_local_agent_container(context, "available") agent_id = context.agent['id'] existing_agents = get_agent(token, agent_id) assert_that(len(existing_agents), greater_than(0), "Agent not created") timeout = 30 agent_status = wait_until_expected_agent_status(token, agent_id, status, timeout=timeout) context.agent = get_agent(token, agent_id) logs = get_orb_agent_logs(context.container_id) assert_that( agent_status, is_(equal_to(status)), f"Agent did not get '{status}' after {str(timeout)} seconds, but was '{agent_status}'. \n" f"Agent: {json.dumps(context.agent, indent=4)}. \n Logs: {logs}") local_orb_path = configs.get("local_orb_path") agent_schema_path = local_orb_path + "/python-test/features/steps/schemas/agent_schema.json" is_schema_valid = validate_json(context.agent, agent_schema_path) assert_that( is_schema_valid, equal_to(True), f"Invalid agent json. \n Agent = {context.agent}." f"Agent logs: {get_orb_agent_logs(context.container_id)}." f"\nLogs: {logs}")
def create_invalid_sink(context, credential): assert_that( credential, any_of(equal_to('endpoint'), equal_to('username'), equal_to('password')), "Invalid prometheus field") check_prometheus_grafana_credentials(context) sink_label_name = sink_label_name_prefix + random_string(10) token = context.token prometheus_credentials = { 'endpoint': context.remote_prometheus_endpoint, 'username': context.prometheus_username, 'password': context.prometheus_key } prometheus_credentials[credential] = prometheus_credentials[ credential][:-2] context.sink = create_new_sink(token, sink_label_name, prometheus_credentials['endpoint'], prometheus_credentials['username'], prometheus_credentials['password']) local_orb_path = configs.get("local_orb_path") sink_schema_path = local_orb_path + "/python-test/features/steps/schemas/sink_schema.json" is_schema_valid = validate_json(context.sink, sink_schema_path) assert_that(is_schema_valid, equal_to(True), f"Invalid sink json. \n Sink = {context.sink}") context.existent_sinks_id.append(context.sink['id'])
def check_agent_backend_pktvisor_routes(context, route): assert_that( route, any_of(equal_to("taps"), equal_to("handlers"), equal_to("inputs"), equal_to("backends")), "Invalid agent route") agent_backend_routes = { "backends": "backends", "taps": "backends/pktvisor/taps", "inputs": "backends/pktvisor/inputs", "handlers": "backends/pktvisor/handlers" } response = requests.get( orb_url + '/api/v1/agents/' + agent_backend_routes[route], headers={'Authorization': f'Bearer {context.token}'}) assert_that( response.status_code, equal_to(200), f"Request to get {route} route failed with status =" + str(response.status_code)) local_orb_path = configs.get("local_orb_path") route_schema_path = local_orb_path + f"/python-test/features/steps/schemas/{route}_schema.json" is_schema_valid = validate_json(response.json(), route_schema_path) assert_that(is_schema_valid, equal_to(True), f"Invalid route json. \n Route = {route}")
def divide_data(dfile, ofolder): jdata = load_json(dfile) if not validate_json(jdata): print("Invalid JSON data: ignoring %s" % dfile) return False n = len(jdata[0]) assert len(jdata) == 2 and len(jdata[1]) == n #identify all dims int_ixs, int_vals = [], [] for i in range(n): var = jdata[0][i] val = jdata[1][i] if is_int(val) and val >= 2: to_divide_ixs = get_arrays_to_divide(n, jdata, val) if to_divide_ixs is not None: jd1 = copy.deepcopy(jdata) jd2 = copy.deepcopy(jdata) jd1[1][i] = int(val / 2) jd2[1][i] = val - int(val / 2) for j in to_divide_ixs: jd1[1][j] = jd1[1][j][:int(val / 2)] jd2[1][j] = jd2[1][j][int(val / 2):] write_to_folder(ofolder, jd1, jd2, dfile) return True print("Cannot divide this dataset! :(") print_shape(jdata) return False
def ordoro_madness(api_url): ''' Main function to initiate fetching parsing login data. ''' res = requests.get(api_url, params={}, headers={}, verify=False) if res.ok and utils.validate_json(res.text): master_data = utils.marshal_data(res.json().get("data")) print "# Uniques Total {0}".format(len(master_data.get("uniques"))) for unique in master_data.get("uniques"): print unique print "# domains with > 1 user" for domain in master_data.get("domains"): print domain print "# April Logins {0}".format(len(master_data.get("april_logins"))) for login in master_data.get("april_logins"): print login else: logging.error("Bad Request or Invalid json.") raise return master_data
def test_validate_json_invalid_json(): """ Unit test to check the validate_json function with invalid json_object inputs """ json_object = {"sku": ""} with pytest.raises(AMLConfigurationException): assert validate_json(data=json_object, schema=parameters_schema, input_name="PARAMETERS_FILE")
async def main(): sub = await get_redis_client() pub = await get_redis_client() (pattern, ) = await sub.subscribe( config["CHANNELS"]["CAM-RES"] ) # read all channels prefixed with `SOME_` while await pattern.wait_message(): data = await pattern.get() validate_json(json.loads(data)) message = json.loads(data.decode()) img = base64.b64decode(message["data"].encode()) img = np.frombuffer(img, dtype=np.float64) # run model inference here print(img.tolist()[0:30]) # change out message once inference is implemented await pub.publish(config["CHANNELS"]["FC-OUT"], data) await asyncio.sleep(1)
def test_validate_json_incorrect_field(): """ Unit test to check if field incorrect """ json_object = { "name": "workspace-name", "resour_group": "resource-group-name", } with pytest.raises(AMLConfigurationException): assert validate_json(data=json_object, schema=parameters_schema, input_name="PARAMETERS_FILE")
async def main(): pub = await get_redis_client() loop = asyncio.get_event_loop() while True: if TEST: await asyncio.sleep(5) data = config["TEST-MSG"]["IMG-TEST"] else: data = await loop.run_in_executor(None, device.read_until) msg = json.loads(data) if validate_json(msg): await pub.publish(config["CHANNELS"]["FC-IN"], data) else: print(f"Invalid serial in {msg}")
def create_sink(context): sink_label_name = sink_label_name_prefix + random_string(10) token = context.token endpoint = context.remote_prometheus_endpoint username = context.prometheus_username password = context.prometheus_key context.sink = create_new_sink(token, sink_label_name, endpoint, username, password) local_orb_path = configs.get("local_orb_path") sink_schema_path = local_orb_path + "/python-test/features/steps/schemas/sink_schema.json" is_schema_valid = validate_json(context.sink, sink_schema_path) assert_that(is_schema_valid, equal_to(True), f"Invalid sink json. \n Sink = {context.sink}") context.existent_sinks_id.append(context.sink['id'])
def function_create(): with utils.AtomicRequest() as atomic: function_id = uuid.uuid4().hex atomic.driver_endpoint = driver_endpoint user, tenant = utils.get_headers(request) zip_file = utils.get_zip(request) zip_url = utils.upload_zip(function_id, zip_file) if not zip_url: atomic.errors = True return critical_error('Not able to store zip.') atomic.zip_url = zip_url metadata = utils.get_metadata(request) if not utils.validate_json(utils.build_schema, metadata): atomic.errors = True return bad_request("Error validating json.") tag = "{0}_{1}_{2}".format(tenant, user, metadata.get('name')) payload = { "memory": metadata.get('memory'), "tags": [tag], "runtime": metadata.get('runtime'), "zip_location": zip_url, "name": metadata.get('name') } image_id = utils.create_image(driver_endpoint, payload) atomic.image_id = image_id function = utils.create_function(tenant, user, function_id, image_id, zip_url, tag, metadata) if not function: atomic.errors = True return critical_error('Error building the function.') return Response(function_id, status=201)
def generate_group_with_valid_json(token, agent_group_name, group_description, tags_to_group, agent_groups): """ Create a group and validate the json schema :param (str) token: used for API authentication :param (str) agent_group_name: of the agent group to be created :param (str) group_description: description of group :param (dict) tags_to_group: dict with all pairs key:value that will be used as tags :returns: (dict) a dictionary containing the created agent group data :return: agent group data """ agent_group_data = create_agent_group(token, agent_group_name, group_description, tags_to_group) group_id = agent_group_data['id'] agent_groups[group_id] = agent_group_name local_orb_path = configs.get("local_orb_path") agent_group_schema_path = local_orb_path + "/python-test/features/steps/schemas/groups_schema.json" is_schema_valid = validate_json(agent_group_data, agent_group_schema_path) assert_that(is_schema_valid, equal_to(True), f"Invalid group json. \n Group = {agent_group_data}") return agent_group_data
def main(): # Loading azure credentials print("::debug::Loading azure credentials") azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}") try: azure_credentials = json.loads(azure_credentials) except JSONDecodeError: print( "::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS" ) raise AMLConfigurationException( "Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md" ) # Checking provided parameters print("::debug::Checking provided parameters") validate_json(data=azure_credentials, schema=azure_credentials_schema, input_name="AZURE_CREDENTIALS") # Mask values print("::debug::Masking parameters") mask_parameter(parameter=azure_credentials.get("tenantId", "")) mask_parameter(parameter=azure_credentials.get("clientId", "")) mask_parameter(parameter=azure_credentials.get("clientSecret", "")) mask_parameter(parameter=azure_credentials.get("subscriptionId", "")) # Loading parameters file print("::debug::Loading parameters file") parameters_file = os.environ.get("INPUT_PARAMETERS_FILE", default="run.json") parameters_file_path = os.path.join(".cloud", ".azure", parameters_file) try: with open(parameters_file_path) as f: parameters = json.load(f) except FileNotFoundError: print( f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/run.json)." ) parameters = {} # Checking provided parameters print("::debug::Checking provided parameters") validate_json(data=parameters, schema=parameters_schema, input_name="PARAMETERS_FILE") # Define target cloud if azure_credentials.get( "resourceManagerEndpointUrl", "").startswith("https://management.usgovcloudapi.net"): cloud = "AzureUSGovernment" elif azure_credentials.get( "resourceManagerEndpointUrl", "").startswith("https://management.chinacloudapi.cn"): cloud = "AzureChinaCloud" else: cloud = "AzureCloud" # Loading Workspace print("::debug::Loading AML Workspace") sp_auth = ServicePrincipalAuthentication( tenant_id=azure_credentials.get("tenantId", ""), service_principal_id=azure_credentials.get("clientId", ""), service_principal_password=azure_credentials.get("clientSecret", ""), cloud=cloud) config_file_path = os.environ.get("GITHUB_WORKSPACE", default=".cloud/.azure") config_file_name = "aml_arm_config.json" try: ws = Workspace.from_config(path=config_file_path, _file_name=config_file_name, auth=sp_auth) except AuthenticationException as exception: print( f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}" ) raise AuthenticationException except AuthenticationError as exception: print(f"::error::Microsoft REST Authentication Error: {exception}") raise AuthenticationError except AdalError as exception: print( f"::error::Active Directory Authentication Library Error: {exception}" ) raise AdalError except ProjectSystemException as exception: print(f"::error::Workspace authorizationfailed: {exception}") raise ProjectSystemException # Create experiment print("::debug::Creating experiment") try: # Default experiment name repository_name = os.environ.get("GITHUB_REPOSITORY").split("/")[-1] branch_name = os.environ.get("GITHUB_REF").split("/")[-1] default_experiment_name = f"{repository_name}-{branch_name}" experiment = Experiment( workspace=ws, name=parameters.get("experiment_name", default_experiment_name)[:36]) except TypeError as exception: experiment_name = parameters.get("experiment", None) print( f"::error::Could not create an experiment with the specified name {experiment_name}: {exception}" ) raise AMLExperimentConfigurationException( f"Could not create an experiment with the specified name {experiment_name}: {exception}" ) except UserErrorException as exception: experiment_name = parameters.get("experiment", None) print( f"::error::Could not create an experiment with the specified name {experiment_name}: {exception}" ) raise AMLExperimentConfigurationException( f"Could not create an experiment with the specified name {experiment_name}: {exception}" ) # Loading run config print("::debug::Loading run config") run_config = None if run_config is None: # Loading run config from runconfig yaml file print("::debug::Loading run config from runconfig yaml file") run_config = load_runconfig_yaml(runconfig_yaml_file=parameters.get( "runconfig_yaml_file", "code/train/run_config.yml")) if run_config is None: # Loading run config from pipeline yaml file print("::debug::Loading run config from pipeline yaml file") run_config = load_pipeline_yaml(workspace=ws, pipeline_yaml_file=parameters.get( "pipeline_yaml_file", "code/train/pipeline.yml")) if run_config is None: # Loading run config from python runconfig file print("::debug::Loading run config from python runconfig file") run_config = load_runconfig_python( workspace=ws, runconfig_python_file=parameters.get("runconfig_python_file", "code/train/run_config.py"), runconfig_python_function_name=parameters.get( "runconfig_python_function_name", "main")) if run_config is None: # Loading values for errors pipeline_yaml_file = parameters.get("pipeline_yaml_file", "code/train/pipeline.yml") runconfig_yaml_file = parameters.get("runconfig_yaml_file", "code/train/run_config.yml") runconfig_python_file = parameters.get("runconfig_python_file", "code/train/run_config.py") runconfig_python_function_name = parameters.get( "runconfig_python_function_name", "main") print( f"::error::Error when loading runconfig yaml definition your repository (Path: /{runconfig_yaml_file})." ) print( f"::error::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_file})." ) print( f"::error::Error when loading python script or function in your repository which defines the experiment config (Script path: '/{runconfig_python_file}', Function: '{runconfig_python_function_name}()')." ) print( "::error::You have to provide either a yaml definition for your run, a yaml definition of your pipeline or a python script, which returns a runconfig (Pipeline, ScriptRunConfig, AutoMlConfig, Estimator, etc.). Please read the documentation for more details." ) raise AMLExperimentConfigurationException( "You have to provide a yaml definition for your run, a yaml definition of your pipeline or a python script, which returns a runconfig. Please read the documentation for more details." ) # Submit run config print("::debug::Submitting experiment config") try: # Defining default tags print("::debug::Defining default tags") default_tags = { "GITHUB_ACTOR": os.environ.get("GITHUB_ACTOR"), "GITHUB_REPOSITORY": os.environ.get("GITHUB_REPOSITORY"), "GITHUB_SHA": os.environ.get("GITHUB_SHA"), "GITHUB_REF": os.environ.get("GITHUB_REF") } run = experiment.submit(config=run_config, tags=dict(parameters.get("tags", {}), **default_tags)) except AzureMLException as exception: print( f"::error::Could not submit experiment config. Your script passed object of type {type(run_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}" ) raise AMLExperimentConfigurationException( f"Could not submit experiment config. Your script passed object of type {type(run_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}" ) except TypeError as exception: print( f"::error::Could not submit experiment config. Your script passed object of type {type(run_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}" ) raise AMLExperimentConfigurationException( f"Could not submit experiment config. Your script passed object of type {type(run_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}" ) # Create outputs print("::debug::Creating outputs") print(f"::set-output name=experiment_name::{run.experiment.name}") print(f"::set-output name=run_id::{run.id}") print(f"::set-output name=run_url::{run.get_portal_url()}") # Waiting for run to complete print("::debug::Waiting for run to complete") if parameters.get("wait_for_completion", True): run.wait_for_completion(show_output=True) # Creating additional outputs of finished run run_metrics = run.get_metrics(recursive=True) print(f"::set-output name=run_metrics::{run_metrics}") run_metrics_markdown = convert_to_markdown(run_metrics) print( f"::set-output name=run_metrics_markdown::{run_metrics_markdown}") # Download artifacts if enabled if parameters.get("download_artifacts", False): # Defining artifacts folder print("::debug::Defining artifacts folder") root_path = os.environ.get("GITHUB_WORKSPACE", default=None) folder_name = f"aml_artifacts_{run.id}" artifact_path = os.path.join(root_path, folder_name) # Downloading artifacts print("::debug::Downloading artifacts") run.download_files( output_directory=os.path.join(artifact_path, "parent")) children = run.get_children(recursive=True) for i, child in enumerate(children): child.download_files( output_directory=os.path.join(artifact_path, f"child_{i}")) # Creating additional outputs print(f"::set-output name=artifact_path::{artifact_path}") # Publishing pipeline print("::debug::Publishing pipeline") if type(run) is PipelineRun and parameters.get("publish_pipeline", False): # Default pipeline name repository_name = os.environ.get("GITHUB_REPOSITORY").split("/")[-1] branch_name = os.environ.get("GITHUB_REF").split("/")[-1] default_pipeline_name = f"{repository_name}-{branch_name}" published_pipeline = run.publish_pipeline( name=parameters.get("pipeline_name", default_pipeline_name), description="Pipeline registered by GitHub Run Action", version=parameters.get("pipeline_version", None), continue_on_step_failure=parameters.get( "pipeline_continue_on_step_failure", False)) # Creating additional outputs print( f"::set-output name=published_pipeline_id::{published_pipeline.id}" ) print( f"::set-output name=published_pipeline_status::{published_pipeline.status}" ) print( f"::set-output name=published_pipeline_endpoint::{published_pipeline.endpoint}" ) elif parameters.get("publish_pipeline", False): print( "::error::Could not register pipeline because you did not pass a pipeline to the action" ) print("::debug::Successfully finished Azure Machine Learning Train Action")
def main(): # Loading azure credentials print("::debug::Loading azure credentials") azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}") try: azure_credentials = json.loads(azure_credentials) except JSONDecodeError: print( "::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS. The JSON should include the following keys: 'tenantId', 'clientId', 'clientSecret' and 'subscriptionId'." ) raise AMLConfigurationException( "Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md" ) # Checking provided parameters print("::debug::Checking provided parameters") validate_json(data=azure_credentials, schema=azure_credentials_schema, input_name="AZURE_CREDENTIALS") # Mask values print("::debug::Masking parameters") mask_parameter(parameter=azure_credentials.get("tenantId", "")) mask_parameter(parameter=azure_credentials.get("clientId", "")) mask_parameter(parameter=azure_credentials.get("clientSecret", "")) mask_parameter(parameter=azure_credentials.get("subscriptionId", "")) # Loading parameters file print("::debug::Loading parameters file") parameters_file = os.environ.get("INPUT_PARAMETERS_FILE", default="compute.json") parameters_file_path = os.path.join(".cloud", ".azure", parameters_file) try: with open(parameters_file_path) as f: parameters = json.load(f) except FileNotFoundError: print( f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/compute.json)." ) parameters = {} # Checking provided parameters print("::debug::Checking provided parameters") validate_json(data=parameters, schema=parameters_schema, input_name="PARAMETERS_FILE") # Define target cloud if azure_credentials.get( "resourceManagerEndpointUrl", "").startswith("https://management.usgovcloudapi.net"): cloud = "AzureUSGovernment" elif azure_credentials.get( "resourceManagerEndpointUrl", "").startswith("https://management.chinacloudapi.cn"): cloud = "AzureChinaCloud" else: cloud = "AzureCloud" # Loading Workspace print("::debug::Loading AML Workspace") sp_auth = ServicePrincipalAuthentication( tenant_id=azure_credentials.get("tenantId", ""), service_principal_id=azure_credentials.get("clientId", ""), service_principal_password=azure_credentials.get("clientSecret", ""), cloud=cloud) config_file_path = os.environ.get("GITHUB_WORKSPACE", default=".cloud/.azure") config_file_name = "aml_arm_config.json" try: ws = Workspace.from_config(path=config_file_path, _file_name=config_file_name, auth=sp_auth) except AuthenticationException as exception: print( f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}" ) raise AuthenticationException except AuthenticationError as exception: print(f"::error::Microsoft REST Authentication Error: {exception}") raise AuthenticationError except AdalError as exception: print( f"::error::Active Directory Authentication Library Error: {exception}" ) raise AdalError except ProjectSystemException as exception: print(f"::error::Workspace authorizationfailed: {exception}") raise ProjectSystemException # Loading compute target try: # Default compute target name repository_name = os.environ.get("GITHUB_REPOSITORY").split( "/")[-1][:16] # names can be max 16 characters print("::debug::Loading existing compute target") compute_target = ComputeTarget(workspace=ws, name=parameters.get( "name", repository_name)) print( f"::debug::Found compute target with same name. Not updating the compute target: {compute_target.serialize()}" ) except ComputeTargetException: print( "::debug::Could not find existing compute target with provided name" ) # Checking provided parameters print("::debug::Checking provided parameters") required_parameters_provided( parameters=parameters, keys=["compute_type"], message= "Required parameter(s) not found in your parameters file for creating a compute target. Please provide a value for the following key(s): " ) print("::debug::Creating new compute target") compute_type = parameters.get("compute_type", "") print(f"::debug::Compute type listed is{compute_type}") if compute_type == "amlcluster": compute_target = create_aml_cluster(workspace=ws, parameters=parameters) print( f"::debug::Successfully created AML cluster: {compute_target.serialize()}" ) elif compute_type == "akscluster": compute_target = create_aks_cluster(workspace=ws, parameters=parameters) print( f"::debug::Successfully created AKS cluster: {compute_target.serialize()}" ) else: print(f"::error::Compute type '{compute_type}' is not supported") raise AMLConfigurationException( f"Compute type '{compute_type}' is not supported.") print( "::debug::Successfully finished Azure Machine Learning Compute Action")
def main(): # Loading azure credentials print("::debug::Loading azure credentials") azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}") try: azure_credentials = json.loads(azure_credentials) except JSONDecodeError: print("::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS") raise AMLConfigurationException( "Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md") # Checking provided parameters print("::debug::Checking provided parameters") validate_json( data=azure_credentials, schema=azure_credentials_schema, input_name="AZURE_CREDENTIALS" ) # Mask values print("::debug::Masking parameters") mask_parameter(parameter=azure_credentials.get("tenantId", "")) mask_parameter(parameter=azure_credentials.get("clientId", "")) mask_parameter(parameter=azure_credentials.get("clientSecret", "")) mask_parameter(parameter=azure_credentials.get("subscriptionId", "")) # Loading parameters file print("::debug::Loading parameters file") parameters_file = os.environ.get( "INPUT_PARAMETERS_FILE", default="run.json") parameters_file_path = os.path.join(".cloud", ".azure", parameters_file) try: with open(parameters_file_path) as f: parameters = json.load(f) except FileNotFoundError: print( f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/run.json).") parameters = {} # Checking provided parameters print("::debug::Checking provided parameters") validate_json( data=parameters, schema=parameters_schema, input_name="PARAMETERS_FILE" ) # Define target cloud if azure_credentials.get("resourceManagerEndpointUrl", "").startswith("https://management.usgovcloudapi.net"): cloud = "AzureUSGovernment" elif azure_credentials.get("resourceManagerEndpointUrl", "").startswith("https://management.chinacloudapi.cn"): cloud = "AzureChinaCloud" else: cloud = "AzureCloud" # Loading Workspace print("::debug::Loading AML Workspace") sp_auth = ServicePrincipalAuthentication( tenant_id=azure_credentials.get("tenantId", ""), service_principal_id=azure_credentials.get("clientId", ""), service_principal_password=azure_credentials.get("clientSecret", ""), cloud=cloud ) config_file_path = os.environ.get( "GITHUB_WORKSPACE", default=".cloud/.azure") config_file_name = "aml_arm_config.json" try: ws = Workspace.from_config( path=config_file_path, _file_name=config_file_name, auth=sp_auth ) except AuthenticationException as exception: print( f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}") raise AuthenticationException except AuthenticationError as exception: print(f"::error::Microsoft REST Authentication Error: {exception}") raise AuthenticationError except AdalError as exception: print( f"::error::Active Directory Authentication Library Error: {exception}") raise AdalError except ProjectSystemException as exception: print(f"::error::Workspace authorizationfailed: {exception}") raise ProjectSystemException # Create experiment print("::debug::Creating experiment") try: # Default experiment name repository_name = os.environ.get("GITHUB_REPOSITORY").split("/")[-1] branch_name = os.environ.get("GITHUB_REF").split("/")[-1] default_experiment_name = f"{repository_name}-{branch_name}" experiment = Experiment( workspace=ws, name=parameters.get("experiment_name", default_experiment_name)[:36] ) except TypeError as exception: experiment_name = parameters.get("experiment", None) print( f"::error::Could not create an experiment with the specified name {experiment_name}: {exception}") raise AMLExperimentConfigurationException( f"Could not create an experiment with the specified name {experiment_name}: {exception}") except UserErrorException as exception: experiment_name = parameters.get("experiment", None) print( f"::error::Could not create an experiment with the specified name {experiment_name}: {exception}") raise AMLExperimentConfigurationException( f"Could not create an experiment with the specified name {experiment_name}: {exception}") # Reading the dataset print("::debug::Reading the dataset") ds_workspace = Workspace( 'aeefecca-6f22-4523-93ba-bd49686ea0ce', 'mshack-azureml', 'ms-hack') train = Dataset.get_by_name(ds_workspace, name='HistogramTrain') test = Dataset.get_by_name(ds_workspace, name='HistogramTest') label_column_name = 'class' # Setting AutoML config print("::debug::Setting AutoML config") automl_settings = { "primary_metric": 'accuracy', "verbosity": logging.INFO, "enable_stack_ensemble": True } compute_target = ComputeTarget(ws, 'githubcluster') automl_config = AutoMLConfig(task='classification', debug_log='automl_errors.log', compute_target=compute_target, training_data=train, validation_data=test, experiment_timeout_hours=.25, label_column_name=label_column_name, **automl_settings ) # Submit run config print("::debug::Submitting experiment config") try: # Defining default tags print("::debug::Defining default tags") default_tags = { "GITHUB_ACTOR": os.environ.get("GITHUB_ACTOR"), "GITHUB_REPOSITORY": os.environ.get("GITHUB_REPOSITORY"), "GITHUB_SHA": os.environ.get("GITHUB_SHA"), "GITHUB_REF": os.environ.get("GITHUB_REF") } run = experiment.submit( automl_config, tags=dict(parameters.get("tags", {}), **default_tags) ) best_run, fitted_model = run.get_output(metric='accuracy') except AzureMLException as exception: print( f"::error::Could not submit experiment config. Your script passed object of type {type(automl_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}") raise AMLExperimentConfigurationException( f"Could not submit experiment config. Your script passed object of type {type(automl_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}") except TypeError as exception: print( f"::error::Could not submit experiment config. Your script passed object of type {type(automl_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}") raise AMLExperimentConfigurationException( f"Could not submit experiment config. Your script passed object of type {type(automl_config)}. Object must be correctly configured and of type e.g. estimator, pipeline, etc.: {exception}") # Create outputs print("::debug::Creating outputs") print(f"::set-output name=experiment_name::{best_run.experiment.name}") print(f"::set-output name=run_id::{best_run.id}") print(f"::set-output name=run_url::{best_run.get_portal_url()}") # Waiting for run to complete print("::debug::Waiting for run to complete") if parameters.get("wait_for_completion", True): run.wait_for_completion(show_output=True) # Creating additional outputs of finished run run_metrics = run.get_metrics() if type( run) is HyperDriveRun else run.get_metrics(recursive=True) # run_metrics = run.get_metrics(recursive=True) # Not working atm because HyperDriveRun thrown error print(f"::set-output name=run_metrics::{run_metrics}") run_metrics_markdown = convert_to_markdown(run_metrics) print( f"::set-output name=run_metrics_markdown::{run_metrics_markdown}") # Download artifacts if enabled if parameters.get("download_artifacts", False): # Defining artifacts folder print("::debug::Defining artifacts folder") root_path = os.environ.get("GITHUB_WORKSPACE", default=None) folder_name = f"aml_artifacts_{run.id}" artifact_path = os.path.join(root_path, folder_name) # Downloading artifacts print("::debug::Downloading artifacts") run.download_files( output_directory=os.path.join(artifact_path, "parent")) children = run.get_children(recursive=True) for i, child in enumerate(children): child.download_files(output_directory=os.path.join( artifact_path, f"child_{i}")) # Creating additional outputs print(f"::set-output name=artifact_path::{artifact_path}") # Publishing pipeline print("::debug::Publishing pipeline") if type(run) is PipelineRun and parameters.get("pipeline_publish", False): # Default pipeline name repository_name = os.environ.get("GITHUB_REPOSITORY").split("/")[-1] branch_name = os.environ.get("GITHUB_REF").split("/")[-1] default_pipeline_name = f"{repository_name}-{branch_name}" published_pipeline = run.publish_pipeline( name=parameters.get("pipeline_name", default_pipeline_name), description="Pipeline registered by GitHub Run Action", version=parameters.get("pipeline_version", None), continue_on_step_failure=parameters.get( "pipeline_continue_on_step_failure", False) ) # Creating additional outputs print( f"::set-output name=published_pipeline_id::{published_pipeline.id}") print( f"::set-output name=published_pipeline_status::{published_pipeline.status}") print( f"::set-output name=published_pipeline_endpoint::{published_pipeline.endpoint}") elif parameters.get("pipeline_publish", False): print("::error::Could not register pipeline because you did not pass a pipeline to the action") print("::debug::Successfully finished Azure Machine Learning Train Action")