def get_cross_validation_hyperdrive_sampler(self) -> GridParameterSampling: if self.perform_sub_fold_cross_validation: return GridParameterSampling(parameter_space={ CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: choice(list(range(self.number_of_cross_validation_splits))), CROSS_VALIDATION_SUB_FOLD_SPLIT_INDEX_TAG_KEY: choice(list(range( self.number_of_cross_validation_splits_per_fold))), }) else: return super().get_cross_validation_hyperdrive_sampler()
def test_get_hyperdrive_config( number_of_cross_validation_splits: int, number_of_cross_validation_splits_per_fold: int, test_output_dirs: TestOutputDirectories) -> None: """ Test to make sure the number of dataset reader workers are set correctly """ if number_of_cross_validation_splits_per_fold > 0: config = HyperDriveTestModelScalar() config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold else: config = HyperDriveTestModelSegmentation() config.number_of_cross_validation_splits = number_of_cross_validation_splits # create HyperDrive config with dummy estimator for testing source_config = SourceConfig(root_folder=test_output_dirs.root_dir, entry_script="something.py", conda_dependencies_files=[]) estimator = Estimator(source_directory=source_config.root_folder, entry_script=source_config.entry_script, compute_target="Local") hd_config = config.get_hyperdrive_config(estimator=estimator) assert hd_config.estimator.source_directory == source_config.root_folder assert hd_config.estimator.run_config.script == source_config.entry_script assert hd_config.estimator._script_params == source_config.script_params if number_of_cross_validation_splits > 0 and number_of_cross_validation_splits_per_fold > 0: assert hd_config._max_total_runs == number_of_cross_validation_splits * \ number_of_cross_validation_splits_per_fold elif number_of_cross_validation_splits > 0: assert hd_config._max_total_runs == number_of_cross_validation_splits else: assert hd_config._max_total_runs == HYPERDRIVE_TOTAL_RUNS if config.perform_cross_validation: # check sampler is as expected sampler = config.get_cross_validation_hyperdrive_sampler() expected_sampler_dict = { CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: choice(list(range(number_of_cross_validation_splits))) } if number_of_cross_validation_splits_per_fold > 0: expected_sampler_dict[ CROSS_VALIDATION_SUB_FOLD_SPLIT_INDEX_TAG_KEY] = choice( list(range(number_of_cross_validation_splits_per_fold))) assert sampler._parameter_space == expected_sampler_dict else: assert vars(config.get_hyperdrive_config(estimator)) \ == vars(_create_dummy_hyperdrive_param_search_config(estimator))
def main(epochs, iterations, compute_target, concurrent_runs): cli_auth = AzureCliAuthentication() experiment = Experiment.from_directory(".", auth=cli_auth) ws = experiment.workspace cluster = ws.compute_targets[compute_target] food_data = ws.datastores['food_images'] script_arguments = {"--data-dir": food_data.as_mount(), "--epochs": epochs} tf_est = TensorFlow(source_directory=".", entry_script='code/train/train.py', script_params=script_arguments, compute_target=cluster, conda_packages=['pillow', 'pandas'], pip_packages=['click', 'seaborn'], use_docker=True, use_gpu=True, framework_version='1.13') # Run on subset of food categories tf_est.run_config.arguments.extend( ['apple_pie', 'baby_back_ribs', 'baklava', 'beef_carpaccio']) param_sampler = RandomParameterSampling({ '--minibatch-size': choice(16, 32, 64), '--learning-rate': loguniform(-9, -6), '--optimizer': choice('rmsprop', 'adagrad', 'adam') }) # Create Early Termination Policy etpolicy = BanditPolicy(evaluation_interval=2, slack_factor=0.1) # Create HyperDrive Run Configuration hyper_drive_config = HyperDriveConfig( estimator=tf_est, hyperparameter_sampling=param_sampler, policy=etpolicy, primary_metric_name='acc', primary_metric_goal=PrimaryMetricGoal.MAXIMIZE, max_total_runs=iterations, max_concurrent_runs=concurrent_runs) # Submit the Hyperdrive Run print("Submitting Hyperdrive Run") hd_run = experiment.submit(hyper_drive_config) hd_run.wait_for_completion(raise_on_error=True, show_output=True) print("Finishing Run") best_run = hd_run.get_best_run_by_primary_metric() print(f'##vso[task.setvariable variable=run_id]{best_run.id}')
def get_cross_validation_hyperdrive_sampler(self) -> GridParameterSampling: """ Returns the cross validation sampler, required to sample the entire parameter space for cross validation. """ return GridParameterSampling(parameter_space={ CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: choice(list(range(self.number_of_cross_validation_splits))), })
def test_get_hyperdrive_config(number_of_cross_validation_splits: int, test_output_dirs: OutputFolderForTests) -> None: """ Test to make sure the number of dataset reader workers are set correctly """ config = HyperDriveTestModel() config.number_of_cross_validation_splits = number_of_cross_validation_splits run_config = ScriptRunConfig(source_directory=str( test_output_dirs.root_dir), script=str(Path("something.py")), arguments=["foo"], compute_target="Local") hd_config = config.get_hyperdrive_config(run_config=run_config) if number_of_cross_validation_splits > 0: assert hd_config._max_total_runs == number_of_cross_validation_splits else: assert hd_config._max_total_runs == HYPERDRIVE_TOTAL_RUNS if config.perform_cross_validation: # check sampler is as expected sampler = config.get_cross_validation_hyperdrive_sampler() expected_sampler_dict = { CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: choice(list(range(number_of_cross_validation_splits))) } assert sampler._parameter_space == expected_sampler_dict else: assert vars(config.get_hyperdrive_config(run_config)) \ == vars(_create_dummy_hyperdrive_param_search_config(run_config))
def exec_SubmitAutoMLTask(self, Parameters: SubmitAutoMLTaskParameter): execResult = False old_stdout = sys.stdout sys.stdout = mystdout = StringIO() if self.experiment is None: print('experiment is None, Please Setup Experiment first.') else: try: image_config = AutoMLImageConfig( task=Parameters.TaskType, compute_target=self.compute_target, training_data=self.training_dataset, validation_data=self.validation_dataset, hyperparameter_sampling=GridParameterSampling( {'model_name': choice(Parameters.ModelChoice)})) self.automl_task_run = self.experiment.submit(image_config) print(self.automl_task_run.get_status()) # self.automl_task_run.wait_for_completion(wait_post_processing=True) execResult = True except Exception as ex: print(ex) sys.stdout = old_stdout return ExecResult(execResult, mystdout.getvalue())
def get_parameter_distribution(distribution, **kwargs): if "choice" in distribution.lower(): parameter_distr = choice( kwargs.get("options", []) ) elif "randint" in distribution.lower(): parameter_distr = randint( upper=kwargs.get("upper", None) ) elif "uniform" in distribution.lower(): parameter_distr = uniform( min_value=kwargs.get("min_value", None), max_value=kwargs.get("max_value", None) ) elif "quniform" in distribution.lower(): parameter_distr = quniform( min_value=kwargs.get("min_value", None), max_value=kwargs.get("max_value", None), q=kwargs.get("q", None) ) elif "loguniform" in distribution.lower(): parameter_distr = loguniform( min_value=kwargs.get("min_value", None), max_value=kwargs.get("max_value", None), ) elif "qloguniform" in distribution.lower(): parameter_distr = qloguniform( min_value=kwargs.get("min_value", None), max_value=kwargs.get("max_value", None), q=kwargs.get("q", None) ) elif "normal" in distribution.lower(): parameter_distr = normal( mu=kwargs.get("mu", None), sigma=kwargs.get("sigma", None) ) elif "qnormal" in distribution.lower(): parameter_distr = qnormal( mu=kwargs.get("mu", None), sigma=kwargs.get("sigma", None), q=kwargs.get("q", None) ) elif "lognormal" in distribution.lower(): parameter_distr = lognormal( mu=kwargs.get("mu", None), sigma=kwargs.get("sigma", None) ) elif "qlognormal" in distribution.lower(): parameter_distr = qlognormal( mu=kwargs.get("mu", None), sigma=kwargs.get("sigma", None), q=kwargs.get("q", None) ) else: parameter_distr = None raise RunConfigurationException(f"Parameter distribution for parameter not defined in settings. Please choose between \'choice\', \'randint\', \'uniform\', \'quniform\', \'loguniform\', \'qloguniform\', \'normal\', \'qnormal\', \'lognormal\' and \'qlognormal\'") return parameter_distr
def get_parameter_distribution(parameter_name, parameter_setting): if "choice" in parameter_setting["distribution"]: parameter_distr = choice(parameter_setting["parameters"]["options"]) elif "randint" in parameter_setting["distribution"]: parameter_distr = randint( upper=parameter_setting["parameters"]["upper"]) elif "uniform" in parameter_setting["distribution"]: parameter_distr = uniform( min_value=parameter_setting["parameters"]["min_value"], max_value=parameter_setting["parameters"]["max_value"]) elif "quniform" in parameter_setting["distribution"]: parameter_distr = quniform( min_value=parameter_setting["parameters"]["min_value"], max_value=parameter_setting["parameters"]["max_value"], q=parameter_setting["parameters"]["q"]) elif "loguniform" in parameter_setting["distribution"]: parameter_distr = loguniform( min_value=parameter_setting["parameters"]["min_value"], max_value=parameter_setting["parameters"]["max_value"]) elif "qloguniform" in parameter_setting["distribution"]: parameter_distr = qloguniform( min_value=parameter_setting["parameters"]["min_value"], max_value=parameter_setting["parameters"]["max_value"], q=parameter_setting["parameters"]["q"]) elif "normal" in parameter_setting["distribution"]: parameter_distr = normal( mu=parameter_setting["parameters"]["mu"], sigma=parameter_setting["parameters"]["sigma"]) elif "qnormal" in parameter_setting["distribution"]: parameter_distr = qnormal( mu=parameter_setting["parameters"]["mu"], sigma=parameter_setting["parameters"]["sigma"], q=parameter_setting["parameters"]["q"]) elif "lognormal" in parameter_setting["distribution"]: parameter_distr = lognormal( mu=parameter_setting["parameters"]["mu"], sigma=parameter_setting["parameters"]["sigma"]) elif "qlognormal" in parameter_setting["distribution"]: parameter_distr = qlognormal( mu=parameter_setting["parameters"]["mu"], sigma=parameter_setting["parameters"]["sigma"], q=parameter_setting["parameters"]["q"]) else: parameter_distr = None raise RunConfigurationException( "Parameter distribution for parameter {} not defined in settings. Please choose between \'choice\', \'randint\', \'uniform\', \'quniform\', \'loguniform\', \'qloguniform\', \'normal\', \'qnormal\', \'lognormal\' and \'qlognormal\'" .format(parameter_name)) return parameter_distr
def get_cross_validation_hyperdrive_config( self, run_config: ScriptRunConfig) -> HyperDriveConfig: """ Returns a configuration for AzureML Hyperdrive that varies the cross validation split index. Because this adds a val/Loss metric it is important that when subclassing LightningContainer your implementeation of LightningModule logs val/Loss. There is an example of this in HelloRegression's validation_step method. :param run_config: The AzureML run configuration object that training for an individual model. :return: A hyperdrive configuration object. """ return HyperDriveConfig( run_config=run_config, hyperparameter_sampling=GridParameterSampling( parameter_space={ CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY: choice(list(range(self.number_of_cross_validation_splits))) }), primary_metric_name=TrackedMetrics.Val_Loss.value, primary_metric_goal=PrimaryMetricGoal.MINIMIZE, max_total_runs=self.number_of_cross_validation_splits)
est = Estimator( source_directory=source_directory, script_params={"--data-folder": ds.as_mount()}, compute_target=compute_target, pip_packages=GeneralConfig.pip_packages, entry_script=PathsConfig.entry_script, use_gpu=True, custom_docker_image=settings["IMAGE_NAME"], ) if GeneralConfig.hyperdrive: if GeneralConfig.architecture_type == "PretrainedResNet50": hyperparams_space = HyperdriveConfig.pretrained_resnet50_hyperparams_space else: raise NotImplementedError hyperparams_space_format = { parameter: choice(parameter_range) for parameter, parameter_range in hyperparams_space.items() } parameters_sampling = RandomParameterSampling(hyperparams_space_format) policy = BanditPolicy( evaluation_interval=HyperdriveConfig.evaluation_interval, slack_factor=HyperdriveConfig.slack_factor, ) hdc = HyperDriveConfig( estimator=est, hyperparameter_sampling=parameters_sampling, policy=policy, primary_metric_name="Accuracy", primary_metric_goal=PrimaryMetricGoal.MAXIMIZE, max_total_runs=HyperdriveConfig.max_total_runs, max_concurrent_runs=HyperdriveConfig.max_concurrent_runs,
args = [ '--input-data', ds.as_named_input('train_data'), #'--num-topics', 10, '--chunksize', 2000, '--passes', 20, '--iterations', 400 ] src = ScriptRunConfig(source_directory="./topicmodel", script='train.py', arguments=args, compute_target=compute_target, environment=env) param_sampling = RandomParameterSampling({ "--num-topics": choice(5, 10, 15, 20) }) # Submit experiment hd = HyperDriveConfig(run_config=src, hyperparameter_sampling=param_sampling, primary_metric_name="c_v", primary_metric_goal=PrimaryMetricGoal.MAXIMIZE, max_total_runs=100, max_concurrent_runs=4) run = exp.submit(config=hd) run.wait_for_completion(show_output=False)
from azureml.widgets import RunDetails from azureml.train.sklearn import SKLearn from azureml.train.hyperdrive.run import PrimaryMetricGoal from azureml.train.hyperdrive.policy import BanditPolicy from azureml.train.hyperdrive.sampling import RandomParameterSampling from azureml.train.hyperdrive.runconfig import HyperDriveConfig from azureml.train.hyperdrive.parameter_expressions import uniform from azureml.train.estimator import Estimator import os from azureml.train.hyperdrive import normal, uniform, choice # Specify parameter sampler ps = RandomParameterSampling({ "learning_rate": uniform(0.05, 0.1), "batch_size": choice(16, 32, 64, 128) }) # Specify a Policy, check job every 2 iterations policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1) if "training" not in os.listdir(): os.mkdir("./training") # workspaceblobstore is the default blob storage #src.run_config.source_directory_data_store = "workspaceblobstore" # Create a SKLearn estimator for use with train.py est = SKLearn("./training", script_params=None, compute_target=compute_target,
'keras==2.0.8', 'theano', 'tensorflow==1.8.0', 'tensorflow-gpu==1.8.0', 'matplotlib', 'horovod', 'hickle' ], entry_script='train.py', use_gpu=True, node_count=1) # run = exp.submit(est) # print(run) # run.wait_for_completion(show_output=True) ps = RandomParameterSampling({ '--batch_size': choice(2, 4, 8, 16), '--filter_sizes': choice("3 3 3", "4 4 4", "5 5 5"), '--stack_sizes': choice("48 96 192", "36 72 144", "12 24 48"), '--learning_rate': loguniform(-6, -1), '--lr_decay': loguniform(-9, -1) }) policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1) #, delay_evaluation=20) hdc = HyperDriveRunConfig(estimator=est, hyperparameter_sampling=ps,
from azureml.widgets import RunDetails from azureml.train.sklearn import SKLearn from azureml.train.hyperdrive import choice from azureml.train.hyperdrive.run import PrimaryMetricGoal from azureml.train.hyperdrive.policy import BanditPolicy from azureml.train.hyperdrive.policy import TruncationSelectionPolicy from azureml.train.hyperdrive.sampling import RandomParameterSampling from azureml.train.hyperdrive.runconfig import HyperDriveConfig from azureml.train.hyperdrive.parameter_expressions import uniform from azureml.core import ScriptRunConfig import os # Specify parameter sampler # https://xgboost.readthedocs.io/en/latest/parameter.html ps = { '--num_boost_round': choice(5, 10, 20, 50), '--max_depth': choice(3, 5, 8), '--learning_rate': choice(0.001, 0.005, 0.01, 0.05), '--gamma': choice(0, 1, 2), '--reg_lambda': choice(0.1, 1, 2, 5), '--scale_pos_weight': choice(1, 2) } samp = RandomParameterSampling(parameter_space=ps) # Specify a Policy policy = TruncationSelectionPolicy( truncation_percentage=50) # BanditPolicy(slack_factor=0.1) if "training" not in os.listdir(): os.mkdir("./training")
def build_pipeline(dataset, ws, config): print("building pipeline for dataset %s in workspace %s" % (dataset, ws.name)) hostname = socket.gethostname() if hostname == 'wopauliNC6': base_dir = '.' else: base_dir = '.' def_blob_store = ws.get_default_datastore() # folder for scripts that need to be uploaded to Aml compute target script_folder = './scripts' os.makedirs(script_folder, exist_ok=True) shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder) shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder) shutil.copy(os.path.join(base_dir, 'pipelines_build.py'), script_folder) shutil.copy(os.path.join(base_dir, 'train.py'), script_folder) shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder) shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder) shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder) shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder) shutil.copy(os.path.join(base_dir, 'model_registration.py'), script_folder) shutil.copy(os.path.join(base_dir, 'config.json'), script_folder) cpu_compute_name = config['cpu_compute'] try: cpu_compute_target = AmlCompute(ws, cpu_compute_name) print("found existing compute target: %s" % cpu_compute_name) except ComputeTargetException: print("creating new compute target") provisioning_config = AmlCompute.provisioning_configuration( vm_size='STANDARD_D2_V2', max_nodes=4, idle_seconds_before_scaledown=1800) cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name, provisioning_config) cpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) # use get_status() to get a detailed status for the current cluster. print(cpu_compute_target.get_status().serialize()) # choose a name for your cluster gpu_compute_name = config['gpu_compute'] try: gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name) print("found existing compute target: %s" % gpu_compute_name) except ComputeTargetException: print('Creating a new compute target...') provisioning_config = AmlCompute.provisioning_configuration( vm_size='STANDARD_NC6', max_nodes=5, idle_seconds_before_scaledown=1800) # create the cluster gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name, provisioning_config) # can poll for a minimum number of nodes and for a specific timeout. # if no min node count is provided it uses the scale settings for the cluster gpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) # use get_status() to get a detailed status for the current cluster. print(gpu_compute_target.get_status().serialize()) # conda dependencies for compute targets cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"], pip_packages=[ "azure-storage-blob==1.5.0", "hickle==3.4.3", "requests==2.21.0", "sklearn", "pandas==0.24.2", "azureml-sdk==1.0.21", "numpy==1.16.2", "pillow==6.0.0" ]) gpu_cd = CondaDependencies.create(pip_packages=[ "keras==2.0.8", "theano==1.0.4", "tensorflow==1.8.0", "tensorflow-gpu==1.8.0", "hickle==3.4.3", "matplotlib==3.0.3", "seaborn==0.9.0", "requests==2.21.0", "bs4==0.0.1", "imageio==2.5.0", "sklearn", "pandas==0.24.2", "azureml-sdk==1.0.21", "numpy==1.16.2" ]) # Runconfigs cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd) cpu_compute_run_config.environment.docker.enabled = True cpu_compute_run_config.environment.docker.gpu_support = False cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE cpu_compute_run_config.environment.spark.precache_packages = False gpu_compute_run_config = RunConfiguration(conda_dependencies=gpu_cd) gpu_compute_run_config.environment.docker.enabled = True gpu_compute_run_config.environment.docker.gpu_support = True gpu_compute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE gpu_compute_run_config.environment.spark.precache_packages = False print("PipelineData object created") video_data = DataReference(datastore=def_blob_store, data_reference_name="video_data", path_on_datastore=os.path.join( "prednet", "data", "video", dataset)) # Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1. raw_data = PipelineData("raw_video_fames", datastore=def_blob_store) preprocessed_data = PipelineData("preprocessed_video_frames", datastore=def_blob_store) data_metrics = PipelineData("data_metrics", datastore=def_blob_store) data_output = PipelineData("output_data", datastore=def_blob_store) print("DataReference object created") # prepare dataset for training/testing prednet video_decoding = PythonScriptStep( name='decode_videos', script_name="video_decoding.py", arguments=["--input_data", video_data, "--output_data", raw_data], inputs=[video_data], outputs=[raw_data], compute_target=cpu_compute_target, source_directory=script_folder, runconfig=cpu_compute_run_config, allow_reuse=True, hash_paths=['.']) print("video_decode created") # prepare dataset for training/testing recurrent neural network data_prep = PythonScriptStep(name='prepare_data', script_name="data_preparation.py", arguments=[ "--input_data", raw_data, "--output_data", preprocessed_data ], inputs=[raw_data], outputs=[preprocessed_data], compute_target=cpu_compute_target, source_directory=script_folder, runconfig=cpu_compute_run_config, allow_reuse=True, hash_paths=['.']) data_prep.run_after(video_decoding) print("data_prep created") est = TensorFlow(source_directory=script_folder, compute_target=gpu_compute_target, pip_packages=[ 'keras==2.0.8', 'theano', 'tensorflow==1.8.0', 'tensorflow-gpu==1.8.0', 'matplotlib', 'horovod', 'hickle' ], entry_script='train.py', use_gpu=True, node_count=1) ps = RandomParameterSampling({ '--batch_size': choice(2, 4, 8, 16), '--filter_sizes': choice("3, 3, 3", "4, 4, 4", "5, 5, 5"), '--stack_sizes': choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"), '--learning_rate': loguniform(-6, -1), '--lr_decay': loguniform(-9, -1), '--freeze_layers': choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "1", "2", "3"), '--transfer_learning': choice("True", "False") }) policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1, delay_evaluation=20) hdc = HyperDriveRunConfig( estimator=est, hyperparameter_sampling=ps, policy=policy, primary_metric_name='val_loss', primary_metric_goal=PrimaryMetricGoal.MINIMIZE, max_total_runs=5, #100, max_concurrent_runs=5, #10, max_duration_minutes=60 * 6) hd_step = HyperDriveStep(name="train_w_hyperdrive", hyperdrive_run_config=hdc, estimator_entry_script_arguments=[ '--data-folder', preprocessed_data, '--remote_execution' ], inputs=[preprocessed_data], metrics_output=data_metrics, allow_reuse=True) hd_step.run_after(data_prep) registration_step = PythonScriptStep( name='register_model', script_name='model_registration.py', arguments=['--input_dir', data_metrics, '--output_dir', data_output], compute_target=gpu_compute_target, inputs=[data_metrics], outputs=[data_output], source_directory=script_folder, allow_reuse=True, hash_paths=['.']) registration_step.run_after(hd_step) pipeline = Pipeline( workspace=ws, steps=[video_decoding, data_prep, hd_step, registration_step]) print("Pipeline is built") pipeline.validate() print("Simple validation complete") pipeline_name = 'prednet_' + dataset pipeline.publish(name=pipeline_name) return pipeline_name
], compute_target=target, environment=env, ) # set up hyperdrive search space convert_base = lambda x: float(np.log(x)) search_space = { "--learning_rate": hyperdrive.loguniform( convert_base(1e-6), convert_base(5e-2)), # NB. loguniform on [exp(min), exp(max)] "--weight_decay": hyperdrive.uniform(5e-3, 15e-2), "--per_device_train_batch_size": hyperdrive.choice([16, 32]), } hyperparameter_sampling = RandomParameterSampling(search_space) policy = TruncationSelectionPolicy(truncation_percentage=50, evaluation_interval=2, delay_evaluation=0) hyperdrive_config = HyperDriveConfig( run_config=config, hyperparameter_sampling=hyperparameter_sampling, policy=policy, primary_metric_name="eval_matthews_correlation", primary_metric_goal=hyperdrive.PrimaryMetricGoal.MAXIMIZE, max_total_runs=20,
# IN AzureML, we can acheive hyper parameter tuning through an experiment that consists of a hyperdrive run, which initiates a child run # for each hyper parameter combination to be tested. Each child run uses a training script with parameterized hyperparams values to train a model # and logs the target performance metric achieved by the trained model. #for hyperparameter tuning we have to define the search space from azureml.train.hyperdrive import choice, normal param_space = { '--batch_size' : choice(16,32,64), '--learning_rate': normal(10,3) } from azureml.train.hyperdrive import GridParameterSampling #all list would be discrete for grid search param_space = { "--batch_size" : choice(16,32,64), '--learning_rate' : choice(0.01, 0.02, 0.03) } param_sampling = GridParameterSampling(param_space) #distributions can be these type #for discrete: # qnormal,quniform,qlognormal, qloguniform # for continuous:
def build_pipeline(dataset, ws, config): print("building pipeline for dataset %s in workspace %s" % (dataset, ws.name)) base_dir = '.' def_blob_store = ws.get_default_datastore() # folder for scripts that need to be uploaded to Aml compute target script_folder = './scripts' os.makedirs(script_folder, exist_ok=True) shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder) shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder) shutil.copy(os.path.join(base_dir, 'pipelines_create.py'), script_folder) shutil.copy(os.path.join(base_dir, 'train.py'), script_folder) shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder) shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder) shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder) shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder) shutil.copy(os.path.join(base_dir, 'model_registration.py'), script_folder) shutil.copy(os.path.join(base_dir, 'config.json'), script_folder) cpu_compute_name = config['cpu_compute'] try: cpu_compute_target = AmlCompute(ws, cpu_compute_name) print("found existing compute target: %s" % cpu_compute_name) except:# ComputeTargetException: print("creating new compute target") provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', max_nodes=4, idle_seconds_before_scaledown=1800) cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name, provisioning_config) cpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) # use get_status() to get a detailed status for the current cluster. print(cpu_compute_target.get_status().serialize()) # choose a name for your cluster gpu_compute_name = config['gpu_compute'] try: gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name) print("found existing compute target: %s" % gpu_compute_name) except: print('Creating a new compute target...') provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC6', max_nodes=10, idle_seconds_before_scaledown=1800) # create the cluster gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name, provisioning_config) # can poll for a minimum number of nodes and for a specific timeout. # if no min node count is provided it uses the scale settings for the cluster gpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) # use get_status() to get a detailed status for the current cluster. try: print(gpu_compute_target.get_status().serialize()) except BaseException as e: print("Could not get status of compute target.") print(e) # conda dependencies for compute targets cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"], pip_indexurl='https://azuremlsdktestpypi.azureedge.net/sdk-release/Candidate/604C89A437BA41BD942B4F46D9A3591D', pip_packages=["azure-storage-blob==1.5.0", "hickle==3.4.3", "requests==2.21.0", "sklearn", "pandas==0.24.2", "azureml-sdk", "numpy==1.16.2", "pillow==6.0.0"]) # Runconfigs cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd) cpu_compute_run_config.environment.docker.enabled = True cpu_compute_run_config.environment.docker.gpu_support = False cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE cpu_compute_run_config.environment.spark.precache_packages = False print("PipelineData object created") # DataReference to where video data is stored. video_data = DataReference( datastore=def_blob_store, data_reference_name="video_data", path_on_datastore=os.path.join("prednet", "data", "video", dataset)) print("DataReference object created") # Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1. raw_data = PipelineData("raw_video_fames", datastore=def_blob_store) preprocessed_data = PipelineData("preprocessed_video_frames", datastore=def_blob_store) data_metrics = PipelineData("data_metrics", datastore=def_blob_store) data_output = PipelineData("output_data", datastore=def_blob_store) # prepare dataset for training/testing prednet video_decoding = PythonScriptStep( name='decode_videos', script_name="video_decoding.py", arguments=["--input_data", video_data, "--output_data", raw_data], inputs=[video_data], outputs=[raw_data], compute_target=cpu_compute_target, source_directory=script_folder, runconfig=cpu_compute_run_config, allow_reuse=True, hash_paths=['.'] ) print("video_decode step created") # prepare dataset for training/testing recurrent neural network data_prep = PythonScriptStep( name='prepare_data', script_name="data_preparation.py", arguments=["--input_data", raw_data, "--output_data", preprocessed_data], inputs=[raw_data], outputs=[preprocessed_data], compute_target=cpu_compute_target, source_directory=script_folder, runconfig=cpu_compute_run_config, allow_reuse=True, hash_paths=['.'] ) data_prep.run_after(video_decoding) print("data_prep step created") # configure access to ACR for pulling our custom docker image acr = ContainerRegistry() acr.address = config['acr_address'] acr.username = config['acr_username'] acr.password = config['acr_password'] est = Estimator(source_directory=script_folder, compute_target=gpu_compute_target, entry_script='train.py', use_gpu=True, node_count=1, custom_docker_image = "wopauli_1.8-gpu:1", image_registry_details=acr, user_managed=True ) ps = RandomParameterSampling( { '--batch_size': choice(1, 2, 4, 8), '--filter_sizes': choice("3, 3, 3", "4, 4, 4", "5, 5, 5"), '--stack_sizes': choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"), '--learning_rate': loguniform(-6, -1), '--lr_decay': loguniform(-9, -1), '--freeze_layers': choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"), '--transfer_learning': choice("True", "False") } ) policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1, delay_evaluation=10) hdc = HyperDriveConfig(estimator=est, hyperparameter_sampling=ps, policy=policy, primary_metric_name='val_loss', primary_metric_goal=PrimaryMetricGoal.MINIMIZE, max_total_runs=10, max_concurrent_runs=5, max_duration_minutes=60*6 ) hd_step = HyperDriveStep( name="train_w_hyperdrive", hyperdrive_run_config=hdc, estimator_entry_script_arguments=[ '--data-folder', preprocessed_data, '--remote_execution', '--dataset', dataset ], inputs=[preprocessed_data], metrics_output = data_metrics, allow_reuse=True ) hd_step.run_after(data_prep) registration_step = PythonScriptStep( name='register_model', script_name='model_registration.py', arguments=['--input_dir', data_metrics, '--output_dir', data_output], compute_target=cpu_compute_target, inputs=[data_metrics], outputs=[data_output], source_directory=script_folder, allow_reuse=True, hash_paths=['.'] ) registration_step.run_after(hd_step) pipeline = Pipeline(workspace=ws, steps=[video_decoding, data_prep, hd_step, registration_step]) print ("Pipeline is built") pipeline.validate() print("Simple validation complete") pipeline_name = 'prednet_' + dataset published_pipeline = pipeline.publish(name=pipeline_name) schedule = Schedule.create(workspace=ws, name=pipeline_name + "_sch", pipeline_id=published_pipeline.id, experiment_name=pipeline_name, datastore=def_blob_store, wait_for_provisioning=True, description="Datastore scheduler for Pipeline" + pipeline_name, path_on_datastore=os.path.join('prednet/data/video', dataset, 'Train'), polling_interval=1 ) return pipeline_name
# -lognormal # -loguniform #continuous hyperparams #normal #uniform #lognormal #loguniform from azureml.train.hyperdrive import choice, normal param_sapce = { '--batch_size': choice(10,12,15), '--learning_rate': normal(10,3) } #configuring sample - grid,random, baysian from azureml.train.hyperdrive import GridParameterSampling, RandomParameterSampling, BayesianParameterSampling param_sampling = GridParameterSampling(param_sapce) #configuring the early stopping - bandit policy -> BanditPolicy(slack_amount = 0.2, evaluation_interval = 1, delay_evaluation = 5)
script='training03.py', arguments=[ '--max_depth', 5, '--num_leaves', 50, '--subsample', 0.9, '--learning_rate', 0.01, '--min_data_in_leaf', 50, '--lambda_l1', 20, '--lambda_l2', 20, '--n_estimators', 1000 ], environment=sklearn_env, compute_target=training_cluster) # Sample a range of parameter values params = BayesianParameterSampling({ # There's only one parameter, so grid sampling will try each value - with multiple parameters it would try every combination '--max_depth': choice(list(range(2, 20))), '--num_leaves': choice(list(range(6, 251))), '--subsample': uniform(0.5, 1), '--learning_rate': uniform(0.005, 0.25), '--min_data_in_leaf': choice(list(range(2, 501))), '--lambda_l1': choice(list(range(201))), '--lambda_l2': choice(list(range(201))), '--n_estimators': choice(list(range(100, 4001, 100))) })
def build_prednet_pipeline(dataset, ws): print("building pipeline for dataset %s in workspace %s" % (dataset, ws.name)) base_dir = "." def_blob_store = ws.get_default_datastore() # folder for scripts that need to be uploaded to Aml compute target script_folder = "./scripts" os.makedirs(script_folder) shutil.copytree(os.path.join(base_dir, "models"), os.path.join(base_dir, script_folder, "models")) shutil.copy(os.path.join(base_dir, "train.py"), script_folder) shutil.copy(os.path.join(base_dir, "data_preparation.py"), script_folder) shutil.copy(os.path.join(base_dir, "register_prednet.py"), script_folder) shutil.copy(os.path.join(base_dir, "batch_scoring.py"), script_folder) shutil.copy(os.path.join(base_dir, "train_clf.py"), script_folder) shutil.copy(os.path.join(base_dir, "register_clf.py"), script_folder) cpu_compute_name = args.cpu_compute_name cpu_compute_target = AmlCompute(ws, cpu_compute_name) print("found existing compute target: %s" % cpu_compute_name) # use get_status() to get a detailed status for the current cluster. print(cpu_compute_target.get_status().serialize()) # choose a name for your cluster gpu_compute_name = args.gpu_compute_name gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name) print(gpu_compute_target.get_status().serialize()) env = Environment.get(ws, "prednet") # Runconfigs runconfig = RunConfiguration() runconfig.environment = env print("PipelineData object created") # DataReference to where raw data is stored. raw_data = DataReference( datastore=def_blob_store, data_reference_name="raw_data", path_on_datastore=os.path.join("prednet", "data", "raw_data"), ) print("DataReference object created") # Naming the intermediate data as processed_data and assigning it to the # variable processed_data. preprocessed_data = PipelineData("preprocessed_data", datastore=def_blob_store) data_metrics = PipelineData("data_metrics", datastore=def_blob_store) hd_child_cwd = PipelineData("prednet_model_path", datastore=def_blob_store) # prednet_path = PipelineData("outputs", datastore=def_blob_store) scored_data = PipelineData("scored_data", datastore=def_blob_store) model_path = PipelineData("model_path", datastore=def_blob_store) # prepare dataset for training/testing recurrent neural network data_prep = PythonScriptStep( name="prepare_data", script_name="data_preparation.py", arguments=[ "--raw_data", raw_data, "--preprocessed_data", preprocessed_data, "--dataset", dataset, ], inputs=[raw_data], outputs=[preprocessed_data], compute_target=cpu_compute_target, source_directory=script_folder, runconfig=runconfig, allow_reuse=True, ) # data_prep.run_after(video_decoding) print("data_prep step created") est = Estimator( source_directory=script_folder, compute_target=gpu_compute_target, entry_script="train.py", node_count=1, environment_definition=env, ) ps = BayesianParameterSampling({ "--batch_size": choice(1, 2, 4, 10), "--filter_sizes": choice("3, 3, 3", "4, 4, 4", "5, 5, 5"), "--stack_sizes": choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), "--learning_rate": uniform(1e-6, 1e-3), "--lr_decay": uniform(1e-9, 1e-2), "--freeze_layers": choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"), # "--fine_tuning": choice("True", "False"), }) hdc = HyperDriveConfig( estimator=est, hyperparameter_sampling=ps, primary_metric_name="val_loss", primary_metric_goal=PrimaryMetricGoal.MINIMIZE, max_total_runs=3, max_concurrent_runs=3, max_duration_minutes=60 * 6, ) train_prednet = HyperDriveStep( "train_w_hyperdrive", hdc, estimator_entry_script_arguments=[ "--preprocessed_data", preprocessed_data, "--remote_execution", "--dataset", dataset, ], inputs=[preprocessed_data], outputs=[hd_child_cwd], metrics_output=data_metrics, allow_reuse=True, ) train_prednet.run_after(data_prep) register_prednet = PythonScriptStep( name="register_prednet", script_name="register_prednet.py", arguments=[ "--data_metrics", data_metrics, ], compute_target=cpu_compute_target, inputs=[data_metrics, hd_child_cwd], source_directory=script_folder, allow_reuse=True, ) register_prednet.run_after(train_prednet) batch_scoring = PythonScriptStep( name="batch_scoring", script_name="batch_scoring.py", arguments=[ "--preprocessed_data", preprocessed_data, "--scored_data", scored_data, "--dataset", dataset, # "--prednet_path", # prednet_path ], compute_target=gpu_compute_target, inputs=[preprocessed_data], outputs=[scored_data], source_directory=script_folder, runconfig=runconfig, allow_reuse=True, ) batch_scoring.run_after(register_prednet) train_clf = PythonScriptStep( name="train_clf", script_name="train_clf.py", arguments=[ "--preprocessed_data", preprocessed_data, "--scored_data", scored_data, "--model_path", model_path ], compute_target=cpu_compute_target, inputs=[preprocessed_data, scored_data], outputs=[model_path], source_directory=script_folder, runconfig=runconfig, allow_reuse=True, ) train_clf.run_after(batch_scoring) register_clf = PythonScriptStep( name="register_clf", script_name="register_clf.py", arguments=["--model_path", model_path], inputs=[model_path], compute_target=cpu_compute_target, source_directory=script_folder, allow_reuse=True, runconfig=runconfig, ) register_clf.run_after(train_clf) pipeline = Pipeline( workspace=ws, steps=[ data_prep, train_prednet, register_prednet, batch_scoring, train_clf, register_clf, ], ) pipeline.validate() pipeline_name = "prednet_" + dataset published_pipeline = pipeline.publish(name=pipeline_name) _ = Schedule.create( workspace=ws, name=pipeline_name + "_sch", pipeline_id=published_pipeline.id, experiment_name=pipeline_name, datastore=def_blob_store, wait_for_provisioning=True, description="Datastore scheduler for Pipeline" + pipeline_name, path_on_datastore=os.path.join("prednet/data/raw_data", dataset, "Train"), polling_interval=60 * 24, ) published_pipeline.submit(ws, pipeline_name)
#%% # Create Experiment object - this will be used to submit the Hyperdrive run and store all the given parameters experiment_hd = Experiment(workspace=ws, name='hyperdrive') #%% [markdown] ###### Create Random Parameter Sampler #%% # Parameter space to sweep over - uses Random Parameter Sampling ps = hd.RandomParameterSampling({ '--network-name': hd.choice('densenet201', 'resnet152', 'resnet34', 'alexnet', 'vgg19_bn'), '--minibatch-size': hd.choice(8, 16), '--learning-rate': hd.uniform(0.00001, 0.001), '--step-size': hd.choice(10, 25, 50), # How often should the learning rate decay '--gamma': hd.uniform( 0.7, 0.99 ), # The decay applied to the learning rate every {step-size} steps '--optimizer-type': hd.choice('sgd', 'adam') }) #%% [markdown]
from azureml.core import ScriptRunConfig, Experiment script_run = ScriptRunConfig(source_directory = ".", script = "hyperdrive_script.py", arguments = ["--input_data", input_ds.as_named_input("raw_data")], environment = my_env, compute_taret = cluster ) #creating hyper parmas from azureml.train.hyperdrive import GridParameterSampling, choice hyper_params = GridParameterSampling({ '--n_estimators': choice(10,20,30,100), '--min_samples_leaf': choice(1,2,5) }) #configuring hyperdrive class from azureml.train.hyperdrive import HyperDriveConfig,PrimaryMetricGoal hyper_config = HyperDriveConfig(run_config=script_run, hyperparameter_sampling = hyper_params, policy= None, primary_metric_name = 'accuray', primary_metric_goal = PrimaryMetricGoal.MAXIMIZE, max_total_runs = 20, max_concurrent_runs=2)
pred = imported_model(X_test) y_hat = np.argmax(pred, axis=1) # print the first 30 labels and predictions print('labels: \t', y_test[:30]) print('predictions:\t', y_hat[:30]) print("Accuracy on the test set:", np.average(y_hat == y_test)) from azureml.train.hyperdrive import RandomParameterSampling, BanditPolicy, HyperDriveConfig, PrimaryMetricGoal from azureml.train.hyperdrive import choice, loguniform ps = RandomParameterSampling({ '--batch-size': choice(32, 64, 128), '--first-layer-neurons': choice(16, 64, 128, 256, 512), '--second-layer-neurons': choice(16, 64, 256, 512), '--learning-rate': loguniform(-6, -1) }) est = TensorFlow(source_directory=script_folder, script_params={ '--data-folder': dataset.as_named_input('mnist').as_mount() }, compute_target=compute_target, entry_script='tf_mnist2.py',
conda_packages=['cudatoolkit=10.0.130'], entry_script='kd_squeezenet.py', use_gpu=True, node_count=1) from azureml.train.hyperdrive import RandomParameterSampling, BanditPolicy, HyperDriveConfig, PrimaryMetricGoal from azureml.pipeline.steps import HyperDriveStep from azureml.train.hyperdrive import choice, loguniform, uniform ps = RandomParameterSampling({ '--learning_rate': uniform(1e-3, 2e-2), '--momentum': uniform(.1, .95), '--weight_decay': loguniform(-5, -3), '--temperature': uniform(1, 9), # '--lambda_const': uniform(.1, .3), '--transfer_learning': choice("True", "False") }) policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1, delay_evaluation=10) hdc = HyperDriveConfig( estimator=est, hyperparameter_sampling=ps, policy=policy, primary_metric_name='val_loss', primary_metric_goal=PrimaryMetricGoal.MINIMIZE, max_total_runs=5, #100, max_concurrent_runs=5)
experiment = Experiment(workspace=ws, name=experiment_name) # Create a folder for the experiment files experiment_folder = './' + experiment_name os.makedirs(experiment_folder, exist_ok=True) print("Experiment:", experiment.name) #Fetch GPU cluster for computations gpu_cluster = ComputeTarget(workspace=ws, name='demo-GPU-cluster') # Sample a range of parameter values params = GridParameterSampling({ # There's only one parameter, so grid sampling will try each value - with multiple parameters it would try every combination '--regularization': choice(0.001, 0.005, 0.01, 0.05, 0.1, 1.0) }) # Set evaluation policy to stop poorly performing training runs early policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1) # Get the training dataset diabetes_ds = ws.datasets.get("diabetes_dataset") # Create an estimator that uses the remote compute hyper_estimator = SKLearn( source_directory=experiment_folder, inputs=[diabetes_ds.as_named_input('diabetes') ], # Pass the dataset as an input compute_target=gpu_cluster, conda_packages=['pandas', 'ipykernel', 'matplotlib'],
path=ds.path('odFridgeObjects/validation_annotations.jsonl')) validation_dataset = validation_dataset.register( workspace=ws, name=validation_dataset_name) print("Training dataset name: " + training_dataset.name) print("Validation dataset name: " + validation_dataset.name) training_dataset.to_pandas_dataframe() image_config_yolov5 = AutoMLImageConfig( task='image-object-detection', compute_target=compute_target, training_data=training_dataset, validation_data=validation_dataset, hyperparameter_sampling=GridParameterSampling( {'model_name': choice('yolov5')})) automl_image_run = experiment.submit(image_config_yolov5) automl_image_run.wait_for_completion(wait_post_processing=True) parameter_space = { 'model': choice( { 'model_name': choice('yolov5'), 'learning_rate': uniform(0.0001, 0.01), #'model_size': choice('small', 'medium'), # model-specific 'img_size': choice(640, 704, 768), # model-specific }, {
if config.get('type') == 'classification': script_params = { '--task': int(task), '--use_cuda': '', '--register_model': '' } est = PyTorch(source_directory=script_folder, compute_target=compute_target, script_params=script_params, entry_script='src/classification.py', pip_packages=pip_packages, use_gpu=True) ### Hyperparameters params if language == 'en': model_type = choice('roberta', 'bert', 'albert') elif language == 'de': model_type = choice('distilbert', 'bert', 'roberta') elif language == 'it' or language == 'es': model_type = choice('bert') elif language == 'fr': model_type = choice('camembert', 'bert') param_sampling = RandomParameterSampling({ '--n_epochs': choice(3, 5, 10), '--learning_rate': choice(1e-5, 2e-5, 3e-5, 4e-5), '--model_type': model_type, '--max_seq_len': choice(128, 256),
print("Training dataset name: " + training_dataset.name) print("Validation dataset name: " + validation_dataset.name) training_dataset.to_pandas_dataframe() from azureml.train.automl import AutoMLImageConfig from azureml.train.hyperdrive import GridParameterSampling from azureml.train.hyperdrive import choice image_config_maskrcnn = AutoMLImageConfig( task='image-instance-segmentation', compute_target=compute_target, training_data=training_dataset, validation_data=validation_dataset, hyperparameter_sampling=GridParameterSampling( {'model_name': choice('maskrcnn_resnet50_fpn')})) automl_image_run = experiment.submit(image_config_maskrcnn) automl_image_run.wait_for_completion(wait_post_processing=True) from azureml.train.automl import AutoMLImageConfig from azureml.train.hyperdrive import GridParameterSampling, RandomParameterSampling, BayesianParameterSampling from azureml.train.hyperdrive import BanditPolicy, HyperDriveConfig, PrimaryMetricGoal from azureml.train.hyperdrive import choice, uniform parameter_space = { 'model_name': choice('maskrcnn_resnet50_fpn'), 'learning_rate': uniform(0.0001, 0.001), #'warmup_cosine_lr_warmup_epochs': choice(0, 3), 'optimizer': choice('sgd', 'adam', 'adamw'),
#define serach space from azureml.train.hyperdrive import choice, normal param_space = { '--batch_size': choice(16, 32, 64), '--learning_rate': normal(10, 3) } #grid sampling example from azureml.train.hyperdrive import GridParameterSampling, choice param_space = { '--batch_size': choice(16, 32, 64), '--learning_rate': choice(0.01, 0.1, 1.0) } param_sampling = GridParameterSampling(param_space) #random sampling from azureml.train.hyperdrive import RandomParameterSampling, choice, normal param_space = { '--batch_size': choice(16, 32, 64), '--learning_rate': normal(10, 3) } param_sampling = RandomParameterSampling(param_space) #Bayesian sampling