def visualize(hdfs_root_logdir): """ Visualize all TensorBoard events for a given path in HopsFS. This is intended for use after running TensorFlow jobs to visualize them all in the same TensorBoard. tflauncher.launch returns the path in HopsFS which should be handed as argument for this method to visualize all runs. Args: :hdfs_root_logdir: the path in HopsFS to enter as the logdir for TensorBoard """ sc = util._find_spark().sparkContext app_id = str(sc.applicationId) pypath = os.getenv("PYSPARK_PYTHON") logdir = os.getcwd() + '/tensorboard_events/' if os.path.exists(logdir): shutil.rmtree(logdir) os.makedirs(logdir) else: os.makedirs(logdir) #find free port tb_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tb_socket.bind(('', 0)) tb_addr, tb_port = tb_socket.getsockname() tb_path = util._find_tensorboard() tb_socket.close() tb_env = os.environ.copy() tb_env['CUDA_VISIBLE_DEVICES'] = '' tb_env['LC_ALL'] = 'C' tb_proc = subprocess.Popen([ pypath, tb_path, "--logdir=%s" % logdir, "--port=%d" % tb_port, "--host=%s" % "0.0.0.0" ], env=tb_env, preexec_fn=util._on_executor_exit('SIGTERM')) host = socket.gethostname() tb_url = "http://{0}:{1}".format(host, tb_port) tb_endpoint = hopshdfs._get_experiments_dir( ) + "/" + app_id + "/TensorBoard.visualize" #dump tb host:port to hdfs pydoop.hdfs.dump(tb_url, tb_endpoint, user=hopshdfs.project_user()) handle = hopshdfs.get() hdfs_logdir_entries = handle.list_directory(hdfs_root_logdir) for entry in hdfs_logdir_entries: file_name, extension = splitext(entry['name']) if not extension == '.log': pydoop.hdfs.get(entry['name'], logdir) tb_proc.wait() stdout, stderr = tb_proc.communicate() print(stdout) print(stderr)
def write_featureframe(self): """ Writes a dataframe of data as a training dataset on HDFS in the petastorm format Returns: None Raises: :ValueError: if not petastorm schema was provided """ spark = util._find_spark() if constants.PETASTORM_CONFIG.SCHEMA in self.petastorm_args: schema = self.petastorm_args[constants.PETASTORM_CONFIG.SCHEMA] del self.petastorm_args[constants.PETASTORM_CONFIG.SCHEMA] else: raise ValueError( "Required petastorm argument 'schema' is not defined in petastorm_args dict" ) if constants.PETASTORM_CONFIG.FILESYSTEM_FACTORY in self.petastorm_args: filesystem_factory = self.petastorm_args[ constants.PETASTORM_CONFIG.FILESYSTEM_FACTORY] del self.petastorm_args[ constants.PETASTORM_CONFIG.FILESYSTEM_FACTORY] else: filesystem_factory = lambda: pa.hdfs.connect( driver=constants.PETASTORM_CONFIG.LIBHDFS) with materialize_dataset(spark, self.path, schema, filesystem_factory=filesystem_factory, **self.petastorm_args): self.df.write.mode(self.write_mode).parquet(self.path)
def num_executors(sc): """ Get the number of executors configured for Jupyter :param sc: The SparkContext to take the executors from. :type sc: [SparkContext :return: Number of configured executors for Jupyter :rtype: int """ sc = hopsutil._find_spark().sparkContext try: return int(sc._conf.get("spark.dynamicAllocation.maxExecutors")) except: # noqa: E722 raise RuntimeError( "Failed to find spark.dynamicAllocation.maxExecutors property, \ please select your mode as either Experiment, Parallel \ Experiments or Distributed Training.")
def _set_spark_hadoop_conf(json_content): spark = None if constants.ENV_VARIABLES.SPARK_IS_DRIVER in os.environ: spark = util._find_spark() if spark is not None: sc = spark.sparkContext sc._jsc.hadoopConfiguration().set( constants.S3_CONFIG.S3_CREDENTIAL_PROVIDER_ENV, constants.S3_CONFIG.S3_TEMPORARY_CREDENTIAL_PROVIDER) sc._jsc.hadoopConfiguration().set( constants.S3_CONFIG.S3_ACCESS_KEY_ENV, json_content[constants.REST_CONFIG.JSON_ACCESS_KEY_ID]) sc._jsc.hadoopConfiguration().set( constants.S3_CONFIG.S3_SECRET_KEY_ENV, json_content[constants.REST_CONFIG.JSON_SECRET_KEY_ID]) sc._jsc.hadoopConfiguration().set( constants.S3_CONFIG.S3_SESSION_KEY_ENV, json_content[constants.REST_CONFIG.JSON_SESSION_TOKEN_ID])
def _convert_dataframe_to_spark(dataframe): """ Helper method for converting a user-provided dataframe into a spark dataframe Args: :dataframe: the input dataframe (supported types are spark rdds, spark dataframes, pandas dataframes, python 2D lists, and numpy 2D arrays) Returns: the dataframe convertd to a spark dataframe Raises: :CouldNotConvertDataframe: in case the provided dataframe could not be converted to a spark dataframe """ spark = util._find_spark() if isinstance(dataframe, pd.DataFrame): sc = spark.sparkContext sql_context = SQLContext(sc) return sql_context.createDataFrame(dataframe) if isinstance(dataframe, list): dataframe = np.array(dataframe) if isinstance(dataframe, np.ndarray): if dataframe.ndim != 2: raise CouldNotConvertDataframe( "Cannot convert numpy array that do not have two dimensions to a dataframe. " "The number of dimensions are: {}".format(dataframe.ndim)) num_cols = dataframe.shape[1] dataframe_dict = {} for n_col in list(range(num_cols)): col_name = "col_" + str(n_col) dataframe_dict[col_name] = dataframe[:, n_col] pandas_df = pd.DataFrame(dataframe_dict) sc = spark.sparkContext sql_context = SQLContext(sc) return sql_context.createDataFrame(pandas_df) if isinstance(dataframe, RDD): return dataframe.toDF() if isinstance(dataframe, DataFrame): return dataframe raise CouldNotConvertDataframe( "The provided dataframe type is not recognized. Supported types are: spark rdds, spark dataframes, " "pandas dataframes, python 2D lists, and numpy 2D arrays. The provided dataframe has type: {}" .format(type(dataframe)))
def launch(map_fun, args_dict=None, name='no-name', local_logdir=False, versioned_resources=None, description=None): """ *Experiment* or *Parallel Experiment* Run an Experiment contained in *map_fun* one time with no arguments or multiple times with different arguments if *args_dict* is specified. Example usage: >>> from hops import experiment >>> def train_nn(): >>> import tensorflow >>> from hops import tensorboard >>> logdir = tensorboard.logdir() >>> # code for preprocessing, training and exporting model >>> # optionally return a value for the experiment which is registered in Experiments service >>> experiment.launch(train_nn) Args: :map_fun: The function to run :args_dict: If specified will run the same function multiple times with different arguments, {'a':[1,2], 'b':[5,3]} would run the function two times with arguments (1,5) and (2,3) provided that the function signature contains two arguments like *def func(a,b):* :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: A longer description for the experiment Returns: HDFS path in your project where the experiment is stored """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running. Please call experiment.end() to stop it.") try: global app_id global experiment_json global elastic_id running = True sc = util._find_spark().sparkContext app_id = str(sc.applicationId) launcher.run_id = launcher.run_id + 1 versioned_path = util._version_resources(versioned_resources, launcher._get_logdir(app_id)) experiment_json = None if args_dict: experiment_json = util._populate_experiment(sc, name, 'experiment', 'launcher', launcher._get_logdir(app_id), json.dumps(args_dict), versioned_path, description) else: experiment_json = util._populate_experiment(sc, name, 'experiment', 'launcher', launcher._get_logdir(app_id), None, versioned_path, description) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) retval, tensorboard_logdir = launcher._launch(sc, map_fun, args_dict, local_logdir) util._version_resources(versioned_resources, launcher._get_logdir(app_id)) if retval: experiment_json = util._finalize_experiment(experiment_json, None, retval) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) return tensorboard_logdir experiment_json = util._finalize_experiment(experiment_json, None, None) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) except: _exception_handler() raise finally: #cleanup spark jobs elastic_id +=1 running = False sc.setJobGroup("", "") return tensorboard_logdir
def parameter_server(map_fun, name='no-name', local_logdir=False, description=None, evaluator=False): """ *Distributed Training* Sets up the cluster to run ParameterServerStrategy. TF_CONFIG is exported in the background and does not need to be set by the user themselves. Example usage: >>> from hops import experiment >>> def distributed_training(): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the wrapper function >>> from hops import tensorboard >>> from hops import devices >>> logdir = tensorboard.logdir() >>> ...ParameterServerStrategy(num_gpus_per_worker=devices.get_num_gpus())... >>> experiment.parameter_server(distributed_training, local_logdir=True) Args:f :map_fun: contains the code where you are using ParameterServerStrategy. :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :description: a longer description for the experiment :evaluator: whether to run one of the workers as an evaluator Returns: HDFS path in your project where the experiment is stored and return value from the process running as chief """ num_ps = util.num_param_servers() num_executors = util.num_executors() assert num_ps > 0, "number of parameter servers should be greater than 0" assert num_ps < num_executors, "num_ps cannot be greater than num_executors (i.e. num_executors == num_ps + num_workers)" if evaluator: assert num_executors - num_ps > 2, "number of workers must be atleast 3 if evaluator is set to True" global running if running: raise RuntimeError("An experiment is currently running.") start = time.time() sc = util._find_spark().sparkContext try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) _start_run() hdfs.mkdir(experiment_utils._get_logdir(app_id, run_id)) experiment_json = experiment_utils._populate_experiment( name, 'parameter_server', 'DISTRIBUTED_TRAINING', None, description, app_id, None, None) experiment_json = experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, 'CREATE') logdir, return_dict = ps_impl._run(sc, map_fun, run_id, local_logdir=local_logdir, name=name, evaluator=evaluator) duration = experiment_utils._seconds_to_milliseconds(time.time() - start) experiment_utils._finalize_experiment(experiment_json, None, app_id, run_id, 'FINISHED', duration, logdir, None, None) return logdir, return_dict except: _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - start)) raise finally: _end_run(sc)
def differential_evolution(objective_function, boundary_dict, direction=Direction.MAX, generations=4, population=6, mutation=0.5, crossover=0.7, name='no-name', local_logdir=False, description=None, optimization_key='metric'): """ *Parallel Experiment* Run differential evolution to explore a given search space for each hyperparameter and figure out the best hyperparameter combination. The function is treated as a blackbox that returns a metric for some given hyperparameter combination. The returned metric is used to evaluate how 'good' the hyperparameter combination was. Example usage: >>> from hops import experiment >>> boundary_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> import tensorflow >>> return network.evaluate(learning_rate, layers, dropout) >>> experiment.differential_evolution(train_nn, boundary_dict, direction=Direction.MAX) Returning multiple outputs, including images and logs: >>> from hops import experiment >>> boundary_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the wrapper function >>> from PIL import Image >>> f = open('logfile.txt', 'w') >>> f.write('Starting training...') >>> accuracy, loss = network.evaluate(learning_rate, layers, dropout) >>> img = Image.new(.....) >>> img.save('diagram.png') >>> return {'accuracy': accuracy, 'loss': loss, 'logfile': 'logfile.txt', 'diagram': 'diagram.png'} >>> # Important! Remember: optimization_key must be set when returning multiple outputs >>> experiment.differential_evolution(train_nn, boundary_dict, direction=Direction.MAX, optimization_key='accuracy') Args: :objective_function: the function to run, must return a metric :boundary_dict: a dict where each key corresponds to an argument of *objective_function* and the correspond value should be a list of two elements. The first element being the lower bound for the parameter and the the second element the upper bound. :direction: Direction.MAX to maximize the returned metric, Direction.MIN to minize the returned metric :generations: number of generations :population: size of population :mutation: mutation rate to explore more different hyperparameters :crossover: how fast to adapt the population to the best in each generation :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :description: a longer description for the experiment :optimization_key: When returning a dict, the key name of the metric to maximize or minimize in the dict should be set as this value Returns: HDFS path in your project where the experiment is stored, dict with best hyperparameters and return dict with best metrics """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running.") start = time.time() sc = util._find_spark().sparkContext try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) _start_run() diff_evo_impl.run_id = run_id hdfs.mkdir(experiment_utils._get_logdir(app_id, run_id)) experiment_json = experiment_utils._populate_experiment( name, 'differential_evolution', 'PARALLEL_EXPERIMENTS', json.dumps(boundary_dict), description, app_id, direction, optimization_key) experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, 'CREATE') logdir, best_param, best_metric, return_dict = diff_evo_impl._run( objective_function, boundary_dict, direction=direction, generations=generations, population=population, mutation=mutation, crossover=crossover, cleanup_generations=False, local_logdir=local_logdir, name=name, optimization_key=optimization_key) duration = experiment_utils._seconds_to_milliseconds(time.time() - start) experiment_utils._finalize_experiment( experiment_json, best_metric, app_id, run_id, 'FINISHED', duration, experiment_utils._get_logdir(app_id, run_id), logdir, optimization_key) return logdir, best_param, return_dict except: _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - start)) raise finally: _end_run(sc)
def mirrored(map_fun, name='no-name', local_logdir=False, versioned_resources=None, description=None): """ *Distributed Training* single machine - multiple GPUs Example usage: >>> from hops import experiment >>> def mirrored_training(): >>> import tensorflow >>> from hops import tensorboard >>> from hops import devices >>> logdir = tensorboard.logdir() >>> ...MirroredStrategy()... >>> experiment.mirrored(mirrored_training) Args: :map_fun: contains the code where you are using MirroredStrategy. :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: a longer description for the experiment Returns: HDFS path in your project where the experiment is stored """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running. Please call experiment.end() to stop it.") try: global app_id global experiment_json global elastic_id running = True sc = util._find_spark().sparkContext app_id = str(sc.applicationId) mirrored_impl.run_id = mirrored_impl.run_id + 1 versioned_path = util._version_resources(versioned_resources, mirrored_impl._get_logdir(app_id)) experiment_json = util._populate_experiment(sc, name, 'experiment', 'mirrored', mirrored_impl._get_logdir(app_id), None, versioned_path, description) util._version_resources(versioned_resources, mirrored_impl._get_logdir(app_id)) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) retval, logdir = mirrored_impl._launch(sc, map_fun, local_logdir=local_logdir, name=name) experiment_json = util._finalize_experiment(experiment_json, None, retval) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) except: _exception_handler() raise finally: #cleanup spark jobs elastic_id +=1 running = False sc.setJobGroup("", "") return logdir
def begin(name='no-name', local_logdir=False, versioned_resources=None, description=None): """ Start a custom Experiment, at the end of the experiment call *end(metric)*. *IMPORTANT* - This call should not be combined with other functions in the experiment module, other than *end*. Other experiment functions such as *grid_search* manages the *begin* and *end* functions internally Example usage: >>> from hops import experiment >>> experiment.begin(name='calculate pi') >>> # Code to calculate pi >>> pi = calc_pi() >>> experiment.end(pi) Args: :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: A longer description for the experiment Returns: HDFS path in your project where the experiment is stored """ global running if running: raise RuntimeError("An experiment is currently running. Please call experiment.stop() to stop it.") try: global app_id global experiment_json global elastic_id global run_id global driver_tensorboard_hdfs_path running = True sc = util._find_spark().sparkContext app_id = str(sc.applicationId) run_id = run_id + 1 versioned_path = util._version_resources(versioned_resources, _get_logdir(app_id)) experiment_json = None experiment_json = util._populate_experiment(sc, name, 'experiment', 'begin', _get_logdir(app_id), None, versioned_path, description) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) hdfs_exec_logdir, hdfs_appid_logdir = hopshdfs._create_directories(app_id, run_id, None, 'begin') pydoop.hdfs.dump('', os.environ['EXEC_LOGFILE'], user=hopshdfs.project_user()) hopshdfs._init_logger() driver_tensorboard_hdfs_path,_ = tensorboard._register(hdfs_exec_logdir, hdfs_appid_logdir, 0, local_logdir=local_logdir) except: _exception_handler() raise return driver_tensorboard_hdfs_path
def parameter_server(map_fun, name='no-name', local_logdir=False, versioned_resources=None, description=None): """ *Distributed Training* Sets up the cluster to run ParameterServerStrategy. TF_CONFIG is exported in the background and does not need to be set by the user themselves. Example usage: >>> from hops import experiment >>> def distributed_training(): >>> import tensorflow >>> from hops import tensorboard >>> from hops import devices >>> logdir = tensorboard.logdir() >>> ...ParameterServerStrategy(num_gpus_per_worker=devices.get_num_gpus())... >>> experiment.parameter_server(distributed_training, local_logdir=True) Args: :map_fun: contains the code where you are using ParameterServerStrategy. :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: a longer description for the experiment Returns: HDFS path in your project where the experiment is stored """ num_ps = util.num_param_servers() num_executors = util.num_executors() assert num_ps > 0, "number of parameter servers should be greater than 0" assert num_ps < num_executors, "num_ps cannot be greater than num_executors (i.e. num_executors == num_ps + num_workers)" global running if running: raise RuntimeError("An experiment is currently running. Please call experiment.end() to stop it.") try: global app_id global experiment_json global elastic_id running = True sc = util._find_spark().sparkContext app_id = str(sc.applicationId) ps.run_id = ps.run_id + 1 versioned_path = util._version_resources(versioned_resources, ps._get_logdir(app_id)) experiment_json = util._populate_experiment(sc, name, 'experiment', 'parameter_server', ps._get_logdir(app_id), None, versioned_path, description) util._version_resources(versioned_resources, ps._get_logdir(app_id)) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) retval, logdir = ps._launch(sc, map_fun, local_logdir=local_logdir, name=name) experiment_json = util._finalize_experiment(experiment_json, None, retval) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) except: _exception_handler() raise finally: #cleanup spark jobs elastic_id +=1 running = False sc.setJobGroup("", "") return logdir
def random_search(map_fun, boundary_dict, direction='max', samples=10, name='no-name', local_logdir=False, versioned_resources=None, description=None): """ *Parallel Experiment* Run an Experiment contained in *map_fun* for configured number of random samples controlled by the *samples* parameter. Each hyperparameter is contained in *boundary_dict* with the key corresponding to the name of the hyperparameter and a list containing two elements defining the lower and upper bound. The experiment must return a metric corresponding to how 'good' the given hyperparameter combination is. Example usage: >>> from hops import experiment >>> boundary_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> import tensorflow >>> # code for preprocessing, training and exporting model >>> # mandatory return a value for the experiment which is registered in Experiments service >>> return network.evaluate(learning_rate, layers, dropout) >>> experiment.random_search(train_nn, boundary_dict, samples=14, direction='max') Args: :map_fun: The function to run :boundary_dict: dict containing hyperparameter name and corresponding boundaries, each experiment randomize a value in the boundary range. :direction: If set to 'max' the highest value returned will correspond to the best solution, if set to 'min' the opposite is true :samples: the number of random samples to evaluate for each hyperparameter given the boundaries :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: A longer description for the experiment Returns: HDFS path in your project where the experiment is stored """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError( "An experiment is currently running. Please call experiment.end() to stop it." ) try: global app_id global experiment_json global elastic_id running = True sc = util._find_spark().sparkContext app_id = str(sc.applicationId) r_search.run_id = r_search.run_id + 1 versioned_path = util._version_resources(versioned_resources, r_search._get_logdir(app_id)) experiment_json = None experiment_json = util._populate_experiment( sc, name, 'experiment', 'random_search', r_search._get_logdir(app_id), json.dumps(boundary_dict), versioned_path, description) util._version_resources(versioned_resources, r_search._get_logdir(app_id)) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) tensorboard_logdir, param, metric = r_search._launch( sc, map_fun, boundary_dict, samples, direction=direction, local_logdir=local_logdir) experiment_json = util._finalize_experiment(experiment_json, param, metric) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) return tensorboard_logdir except: _exception_handler() raise finally: #cleanup spark jobs elastic_id += 1 running = False sc.setJobGroup("", "") return tensorboard_logdir
def grid_search(train_fn, grid_dict, direction=Direction.MAX, name='no-name', local_logdir=False, description=None, optimization_key='metric'): """ *Parallel Experiment* Run grid search evolution to explore a predefined set of hyperparameter combinations. The function is treated as a blackbox that returns a metric for some given hyperparameter combination. The returned metric is used to evaluate how 'good' the hyperparameter combination was. Example usage: >>> from hops import experiment >>> grid_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> return network.evaluate(learning_rate, layers, dropout) >>> experiment.grid_search(train_nn, grid_dict, direction=Direction.MAX) Returning multiple outputs, including images and logs: >>> from hops import experiment >>> grid_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> from PIL import Image >>> f = open('logfile.txt', 'w') >>> f.write('Starting training...') >>> accuracy, loss = network.evaluate(learning_rate, layers, dropout) >>> img = Image.new(.....) >>> img.save('diagram.png') >>> return {'accuracy': accuracy, 'loss': loss, 'logfile': 'logfile.txt', 'diagram': 'diagram.png'} >>> # Important! Remember: optimization_key must be set when returning multiple outputs >>> experiment.grid_search(train_nn, grid_dict, direction=Direction.MAX, optimization_key='accuracy') Args: :train_fn: the function to run, must return a metric :grid_dict: a dict with a key for each argument with a corresponding value being a list containing the hyperparameters to test, internally all possible combinations will be generated and run as separate Experiments :direction: Direction.MAX to maximize the returned metric, Direction.MIN to minize the returned metric :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :description: a longer description for the experiment :optimization_key: When returning a dict, the key name of the metric to maximize or minimize in the dict should be set as this value Returns: HDFS path in your project where the experiment is stored, dict with best hyperparameters and return dict with best metrics """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running.") start = time.time() sc = util._find_spark().sparkContext try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) _start_run() experiment_utils._create_experiment_dir(app_id, run_id) experiment_json = experiment_utils._populate_experiment( name, 'grid_search', 'PARALLEL_EXPERIMENTS', json.dumps(grid_dict), description, app_id, direction, optimization_key) experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, 'CREATE') grid_params = experiment_utils.grid_params(grid_dict) logdir, best_param, best_metric, return_dict = grid_search_impl._run( sc, train_fn, run_id, grid_params, direction=direction, local_logdir=local_logdir, name=name, optimization_key=optimization_key) duration = experiment_utils._seconds_to_milliseconds(time.time() - start) experiment_utils._finalize_experiment( experiment_json, best_metric, app_id, run_id, 'FINISHED', duration, experiment_utils._get_logdir(app_id, run_id), logdir, optimization_key) return logdir, best_param, return_dict except: _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - start)) raise finally: _end_run(sc)
def random_search(train_fn, boundary_dict, direction=Direction.MAX, samples=10, name='no-name', local_logdir=False, description=None, optimization_key='metric'): """ *Parallel Experiment* Run an Experiment contained in *train_fn* for configured number of random samples controlled by the *samples* parameter. Each hyperparameter is contained in *boundary_dict* with the key corresponding to the name of the hyperparameter and a list containing two elements defining the lower and upper bound. The experiment must return a metric corresponding to how 'good' the given hyperparameter combination is. Example usage: >>> from hops import experiment >>> boundary_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> return network.evaluate(learning_rate, layers, dropout) >>> experiment.differential_evolution(train_nn, boundary_dict, direction='max') Returning multiple outputs, including images and logs: >>> from hops import experiment >>> boundary_dict = {'learning_rate': [0.1, 0.3], 'layers': [2, 9], 'dropout': [0.1,0.9]} >>> def train_nn(learning_rate, layers, dropout): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> from PIL import Image >>> f = open('logfile.txt', 'w') >>> f.write('Starting training...') >>> accuracy, loss = network.evaluate(learning_rate, layers, dropout) >>> img = Image.new(.....) >>> img.save('diagram.png') >>> return {'accuracy': accuracy, 'loss': loss, 'logfile': 'logfile.txt', 'diagram': 'diagram.png'} >>> # Important! Remember: optimization_key must be set when returning multiple outputs >>> experiment.differential_evolution(train_nn, boundary_dict, direction='max', optimization_key='accuracy') Args: :train_fn: The function to run :boundary_dict: dict containing hyperparameter name and corresponding boundaries, each experiment randomize a value in the boundary range. :direction: Direction.MAX to maximize the returned metric, Direction.MIN to minize the returned metric :samples: the number of random samples to evaluate for each hyperparameter given the boundaries, for example samples=3 would result in 3 hyperparameter combinations in total to evaluate :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :description: A longer description for the experiment :optimization_key: When returning a dict, the key name of the metric to maximize or minimize in the dict should be set as this value Returns: HDFS path in your project where the experiment is stored, dict with best hyperparameters and return dict with best metrics """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running.") start = time.time() sc = util._find_spark().sparkContext try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) _start_run() experiment_utils._create_experiment_dir(app_id, run_id) experiment_json = experiment_utils._populate_experiment( name, 'random_search', 'PARALLEL_EXPERIMENTS', json.dumps(boundary_dict), description, app_id, direction, optimization_key) experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, 'CREATE') logdir, best_param, best_metric, return_dict = r_search_impl._run( sc, train_fn, run_id, boundary_dict, samples, direction=direction, local_logdir=local_logdir, optimization_key=optimization_key) duration = experiment_utils._seconds_to_milliseconds(time.time() - start) experiment_utils._finalize_experiment( experiment_json, best_metric, app_id, run_id, 'FINISHED', duration, experiment_utils._get_logdir(app_id, run_id), logdir, optimization_key) return logdir, best_param, return_dict except: _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - start)) raise finally: _end_run(sc)
def finalize(self, job_end): results = "" if self.experiment_type == "optimization": _ = self.optimizer.finalize_experiment(self._final_store) self.job_end = job_end self.duration = experiment_utils._seconds_to_milliseconds( self.job_end - self.job_start) self.duration_str = experiment_utils._time_diff( self.job_start, self.job_end) results = ("\n------ " + self.optimizer.name() + " Results ------ direction(" + self.direction + ") \n" "BEST combination " + json.dumps(self.result["best_hp"]) + " -- metric " + str(self.result["best_val"]) + "\n" "WORST combination " + json.dumps(self.result["worst_hp"]) + " -- metric " + str(self.result["worst_val"]) + "\n" "AVERAGE metric -- " + str(self.result["avg"]) + "\n" "EARLY STOPPED Trials -- " + str(self.result["early_stopped"]) + "\n" "Total job time " + self.duration_str + "\n") elif self.experiment_type == "ablation": _ = self.ablator.finalize_experiment(self._final_store) self.job_end = job_end self.duration = experiment_utils._seconds_to_milliseconds( self.job_end - self.job_start) self.duration_str = experiment_utils._time_diff( self.job_start, self.job_end) results = ("\n------ " + self.ablator.name() + " Results ------ \n" + "BEST Config Excludes " + json.dumps(self.result["best_config"]) + " -- metric " + str(self.result["best_val"]) + "\n" + "WORST Config Excludes " + json.dumps(self.result["worst_config"]) + " -- metric " + str(self.result["worst_val"]) + "\n" + "AVERAGE metric -- " + str(self.result["avg"]) + "\n" + "Total Job Time " + self.duration_str + "\n") print(results) self._log(results) hopshdfs.dump( json.dumps(self.result, default=util.json_default_numpy), self.log_dir + "/result.json", ) sc = hopsutil._find_spark().sparkContext hopshdfs.dump(self.json(sc), self.log_dir + "/maggy.json") return self.result
def start(self, exp_driver): """ Start listener in a background thread. Returns: address of the Server as a tuple of (host, port) """ global server_host_port server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if not server_host_port: server_sock.bind(("", 0)) # hostname may not be resolvable but IP address probably will be host = experiment_utils._get_ip_address() port = server_sock.getsockname()[1] server_host_port = (host, port) # register this driver with Hopsworks sc = hopsutil._find_spark().sparkContext app_id = str(sc.applicationId) method = hopsconstants.HTTP_CONFIG.HTTP_POST resource_url = (hopsconstants.DELIMITERS.SLASH_DELIMITER + hopsconstants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + hopsconstants.DELIMITERS.SLASH_DELIMITER + "maggy" + hopsconstants.DELIMITERS.SLASH_DELIMITER + "drivers") json_contents = { "hostIp": host, "port": port, "appId": app_id, "secret": exp_driver._secret, } json_embeddable = json.dumps(json_contents) headers = { hopsconstants.HTTP_CONFIG.HTTP_CONTENT_TYPE: hopsconstants.HTTP_CONFIG.HTTP_APPLICATION_JSON } try: response = hopsutil.send_request(method, resource_url, data=json_embeddable, headers=headers) if (response.status_code // 100) != 2: print("No connection to Hopsworks for logging.") exp_driver._log("No connection to Hopsworks for logging.") except Exception as e: print("Connection failed to Hopsworks. No logging.") exp_driver._log(e) exp_driver._log("Connection failed to Hopsworks. No logging.") else: server_sock.bind(server_host_port) server_sock.listen(10) def _listen(self, sock, driver): CONNECTIONS = [] CONNECTIONS.append(sock) while not self.done: read_socks, _, _ = select.select(CONNECTIONS, [], [], 60) for sock in read_socks: if sock == server_sock: client_sock, client_addr = sock.accept() CONNECTIONS.append(client_sock) _ = client_addr else: try: msg = self.receive(sock) # raise exception if secret does not match # so client socket gets closed if not secrets.compare_digest( msg["secret"], exp_driver._secret): exp_driver._log("SERVER secret: {}".format( exp_driver._secret)) exp_driver._log( "ERROR: wrong secret {}".format( msg["secret"])) raise Exception self._handle_message(sock, msg, driver) except Exception as e: _ = e sock.close() CONNECTIONS.remove(sock) server_sock.close() t = threading.Thread(target=_listen, args=(self, server_sock, exp_driver)) t.daemon = True t.start() return server_host_port
def _run(train_fn, search_dict, direction=Direction.MAX, generations=4, population=6, mutation=0.5, crossover=0.7, cleanup_generations=False, local_logdir=False, name="no-name", optimization_key=None): """ Args: :train_fn: :search_dict: :direction: :generations: :population: :mutation: :crossover: :cleanup_generations: :local_logdir: :name: :optimization_key: Returns: """ global run_id global local_logdir_bool local_logdir_bool = local_logdir global spark spark = util._find_spark() global objective_function objective_function = train_fn global cleanup cleanup = cleanup_generations global opt_key opt_key = optimization_key argcount = six.get_function_code(train_fn).co_argcount arg_names = six.get_function_code(train_fn).co_varnames ordered_arr = [] app_id = spark.sparkContext.applicationId arg_lists = list(search_dict.values()) for i in range(len(arg_lists)): if len(arg_lists[i]) != 2: raise ValueError( 'Boundary list must contain exactly two elements, [lower_bound, upper_bound] for float/int ' 'or [category1, category2] in the case of strings') assert population > 3, 'population should be greater than 3' assert generations > 1, 'generations should be greater than 1' argIndex = 0 while argcount != 0: ordered_arr.append( (arg_names[argIndex], search_dict[arg_names[argIndex]])) argcount = argcount - 1 argIndex = argIndex + 1 ordered_dict = OrderedDict(ordered_arr) bounds_list = [] types_list = [] for entry in ordered_dict: bounds_list.append((ordered_dict[entry][0], ordered_dict[entry][1])) if isinstance(ordered_dict[entry][0], int): types_list.append('int') elif isinstance(ordered_dict[entry][0], float): types_list.append('float') else: types_list.append('cat') global diff_evo diff_evo = DifferentialEvolution(_execute_all, bounds_list, types_list, ordered_dict, direction=direction, generations=generations, population=population, crossover=crossover, mutation=mutation, name=name) root_dir = experiment_utils._get_experiments_dir() + "/" + str( app_id) + "_" + str(run_id) best_param, best_metric = diff_evo._solve(root_dir) param_string = '' for hp in best_param: param_string = param_string + hp + '&' param_string = param_string[:-1] best_exp_logdir, return_dict = _get_best(str(root_dir), direction) print('Finished Experiment \n') return best_exp_logdir, experiment_utils._get_params_dict( best_exp_logdir), best_metric, return_dict
def lagom( map_fun, name="no-name", experiment_type="optimization", searchspace=None, optimizer=None, direction="max", num_trials=1, ablation_study=None, ablator=None, optimization_key="metric", hb_interval=1, es_policy="median", es_interval=300, es_min=10, description="", ): """Launches a maggy experiment, which depending on `experiment_type` can either be a hyperparameter optimization or an ablation study experiment. Given a search space, objective and a model training procedure `map_fun` (black-box function), an experiment is the whole process of finding the best hyperparameter combination in the search space, optimizing the black-box function. Currently maggy supports random search and a median stopping rule. **lagom** is a Swedish word meaning "just the right amount". :param map_fun: User defined experiment containing the model training. :type map_fun: function :param name: A user defined experiment identifier. :type name: str :param experiment_type: Type of Maggy experiment, either 'optimization' (default) or 'ablation'. :type experiment_type: str :param searchspace: A maggy Searchspace object from which samples are drawn. :type searchspace: Searchspace :param optimizer: The optimizer is the part generating new trials. :type optimizer: str, AbstractOptimizer :param direction: If set to ‘max’ the highest value returned will correspond to the best solution, if set to ‘min’ the opposite is true. :type direction: str :param num_trials: the number of trials to evaluate given the search space, each containing a different hyperparameter combination :type num_trials: int :param ablation_study: Ablation study object. Can be None for optimization experiment type. :type ablation_study: AblationStudy :param ablator: Ablator to use for experiment type 'ablation'. :type ablator: str, AbstractAblator :param optimization_key: Name of the metric to be optimized :type optimization_key: str, optional :param hb_interval: The heartbeat interval in seconds from trial executor to experiment driver, defaults to 1 :type hb_interval: int, optional :param es_policy: The earlystopping policy, defaults to 'median' :type es_policy: str, optional :param es_interval: Frequency interval in seconds to check currently running trials for early stopping, defaults to 300 :type es_interval: int, optional :param es_min: Minimum number of trials finalized before checking for early stopping, defaults to 10 :type es_min: int, optional :param description: A longer description of the experiment. :type description: str, optional :raises RuntimeError: An experiment is currently running. :return: A dictionary indicating the best trial and best hyperparameter combination with it's performance metric :rtype: dict """ global running if running: raise RuntimeError("An experiment is currently running.") job_start = time.time() sc = hopsutil._find_spark().sparkContext exp_driver = None try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) app_id, run_id = util._validate_ml_id(app_id, run_id) # start run running = True experiment_utils._set_ml_id(app_id, run_id) # create experiment dir experiment_utils._create_experiment_dir(app_id, run_id) tensorboard._register(experiment_utils._get_logdir(app_id, run_id)) num_executors = util.num_executors(sc) # start experiment driver if experiment_type == "optimization": assert num_trials > 0, "number of trials should be greater " + "than zero" tensorboard._write_hparams_config( experiment_utils._get_logdir(app_id, run_id), searchspace ) if num_executors > num_trials: num_executors = num_trials exp_driver = experimentdriver.ExperimentDriver( "optimization", searchspace=searchspace, optimizer=optimizer, direction=direction, num_trials=num_trials, name=name, num_executors=num_executors, hb_interval=hb_interval, es_policy=es_policy, es_interval=es_interval, es_min=es_min, description=description, log_dir=experiment_utils._get_logdir(app_id, run_id), ) exp_function = exp_driver.optimizer.name() elif experiment_type == "ablation": exp_driver = experimentdriver.ExperimentDriver( "ablation", ablation_study=ablation_study, ablator=ablator, name=name, num_executors=num_executors, hb_interval=hb_interval, description=description, log_dir=experiment_utils._get_logdir(app_id, run_id), ) # using exp_driver.num_executor since # it has been set using ablator.get_number_of_trials() # in experiment.py if num_executors > exp_driver.num_executors: num_executors = exp_driver.num_executors exp_function = exp_driver.ablator.name() else: running = False raise RuntimeError( "Unknown experiment_type:" "should be either 'optimization' or 'ablation', " "But it is '{0}'".format(str(experiment_type)) ) nodeRDD = sc.parallelize(range(num_executors), num_executors) # Do provenance after initializing exp_driver, because exp_driver does # the type checks for optimizer and searchspace sc.setJobGroup(os.environ["ML_ID"], "{0} | {1}".format(name, exp_function)) experiment_json = experiment_utils._populate_experiment( name, exp_function, "MAGGY", exp_driver.searchspace.json(), description, app_id, direction, optimization_key, ) experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, "CREATE" ) util._log( "Started Maggy Experiment: {0}, {1}, run {2}".format(name, app_id, run_id) ) exp_driver.init(job_start) server_addr = exp_driver.server_addr # Force execution on executor, since GPU is located on executor nodeRDD.foreachPartition( trialexecutor._prepare_func( app_id, run_id, experiment_type, map_fun, server_addr, hb_interval, exp_driver._secret, optimization_key, experiment_utils._get_logdir(app_id, run_id), ) ) job_end = time.time() result = exp_driver.finalize(job_end) best_logdir = ( experiment_utils._get_logdir(app_id, run_id) + "/" + result["best_id"] ) util._finalize_experiment( experiment_json, float(result["best_val"]), app_id, run_id, "FINISHED", exp_driver.duration, experiment_utils._get_logdir(app_id, run_id), best_logdir, optimization_key, ) util._log("Finished Experiment") return result except: # noqa: E722 _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - job_start) ) if exp_driver: if exp_driver.exception: raise exp_driver.exception raise finally: # grace period to send last logs to sparkmagic # sparkmagic hb poll intervall is 5 seconds, therefore wait 6 seconds time.sleep(6) # cleanup spark jobs if running and exp_driver is not None: exp_driver.stop() run_id += 1 running = False sc.setJobGroup("", "") return result
def launch(train_fn, args_dict=None, name='no-name', local_logdir=False, description=None, metric_key=None): """ *Experiment* or *Parallel Experiment* Run an Experiment contained in *train_fn* one time with no arguments or multiple times with different arguments if *args_dict* is specified. Example usage: >>> from hops import experiment >>> def train_nn(): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> accuracy, loss = network.evaluate(learning_rate, layers, dropout) >>> experiment.launch(train_nn) Returning multiple outputs, including images and logs: >>> from hops import experiment >>> def train_nn(): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> from PIL import Image >>> f = open('logfile.txt', 'w') >>> f.write('Starting training...') >>> accuracy, loss = network.evaluate(learning_rate, layers, dropout) >>> img = Image.new(.....) >>> img.save('diagram.png') >>> return {'accuracy': accuracy, 'loss': loss, 'logfile': 'logfile.txt', 'diagram': 'diagram.png'} >>> experiment.launch(train_nn) Args: :train_fn: The function to run :args_dict: If specified will run the same function multiple times with different arguments, {'a':[1,2], 'b':[5,3]} would run the function two times with arguments (1,5) and (2,3) provided that the function signature contains two arguments like *def func(a,b):* :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :description: A longer description for the experiment :metric_key: If returning a dict with multiple return values, this key should match the name of the key in the dict for the metric you want to associate with the experiment Returns: HDFS path in your project where the experiment is stored """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError( "An experiment is currently running. Please call experiment.end() to stop it." ) start = time.time() sc = util._find_spark().sparkContext try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) _start_run() experiment_utils._create_experiment_dir(app_id, run_id) experiment_json = None if args_dict: experiment_json = experiment_utils._populate_experiment( name, 'launch', 'EXPERIMENT', json.dumps(args_dict), description, app_id, None, None) else: experiment_json = experiment_utils._populate_experiment( name, 'launch', 'EXPERIMENT', None, description, app_id, None, None) experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, 'CREATE') logdir, return_dict = launcher._run(sc, train_fn, run_id, args_dict, local_logdir) duration = experiment_utils._seconds_to_milliseconds(time.time() - start) metric = experiment_utils._get_metric(return_dict, metric_key) experiment_utils._finalize_experiment(experiment_json, metric, app_id, run_id, 'FINISHED', duration, logdir, None, None) return logdir, return_dict except: _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - start)) raise finally: _end_run(sc)
def differential_evolution(objective_function, boundary_dict, direction = 'max', generations=10, population=10, mutation=0.5, crossover=0.7, cleanup_generations=False, name='no-name', local_logdir=False, versioned_resources=None, description=None): """ *Parallel Experiment* Run differential evolution to explore a given search space for each hyperparameter and figure out the best hyperparameter combination. The function is treated as a blackbox that returns a metric for some given hyperparameter combination. The returned metric is used to evaluate how 'good' the hyperparameter combination was. Example usage: >>> from hops import experiment >>> boundary_dict = {'learning_rate':[0.01, 0.2], 'dropout': [0.1, 0.9]} >>> def train_nn(learning_rate, dropout): >>> import tensorflow >>> # code for preprocessing, training and exporting model >>> # mandatory return a value for the experiment which is registered in Experiments service >>> return network.evaluate(learning_rate, dropout) >>> experiment.differential_evolution(train_nn, boundary_dict, direction='max') Args: :objective_function: the function to run, must return a metric :boundary_dict: a dict where each key corresponds to an argument of *objective_function* and the correspond value should be a list of two elements. The first element being the lower bound for the parameter and the the second element the upper bound. :direction: 'max' to maximize the returned metric, 'min' to minize the returned metric :generations: number of generations :population: size of population :mutation: mutation rate to explore more different hyperparameters :crossover: how fast to adapt the population to the best in each generation :cleanup_generations: remove previous generations from HDFS, only keep the last 2 :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: a longer description for the experiment Returns: HDFS path in your project where the experiment is stored, dict with best hyperparameters """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running. Please call experiment.end() to stop it.") try: global app_id global experiment_json global elastic_id running = True spark = util._find_spark() sc = spark.sparkContext app_id = str(sc.applicationId) diff_evo.run_id = diff_evo.run_id + 1 versioned_path = util._version_resources(versioned_resources, diff_evo._get_logdir(app_id)) experiment_json = None experiment_json = util._populate_experiment(sc, name, 'experiment', 'differential_evolution', diff_evo._get_logdir(app_id), json.dumps(boundary_dict), versioned_path, description) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) tensorboard_logdir, best_param, best_metric = diff_evo._search(spark, objective_function, boundary_dict, direction=direction, generations=generations, popsize=population, mutation=mutation, crossover=crossover, cleanup_generations=cleanup_generations, local_logdir=local_logdir, name=name) experiment_json = util._finalize_experiment(experiment_json, best_param, best_metric) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) best_param_dict = util._convert_to_dict(best_param) except: _exception_handler() raise finally: #cleanup spark jobs elastic_id +=1 running = False sc.setJobGroup("", "") return tensorboard_logdir, best_param_dict
def mirrored(train_fn, name='no-name', local_logdir=False, description=None, evaluator=False, metric_key=None): """ *Distributed Training* Example usage: >>> from hops import experiment >>> def mirrored_training(): >>> # Do all imports in the function >>> import tensorflow >>> # Put all code inside the train_fn function >>> from hops import tensorboard >>> from hops import devices >>> logdir = tensorboard.logdir() >>> ...MirroredStrategy()... >>> experiment.mirrored(mirrored_training, local_logdir=True) Args: :train_fn: contains the code where you are using MirroredStrategy. :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :description: a longer description for the experiment :evaluator: whether to run one of the workers as an evaluator :metric_key: If returning a dict with multiple return values, this key should match the name of the key in the dict for the metric you want to associate with the experiment Returns: HDFS path in your project where the experiment is stored and return value from the process running as chief """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running.") num_workers = util.num_executors() if evaluator: assert num_workers > 2, "number of workers must be atleast 3 if evaluator is set to True" start = time.time() sc = util._find_spark().sparkContext try: global app_id global experiment_json global run_id app_id = str(sc.applicationId) _start_run() experiment_utils._create_experiment_dir(app_id, run_id) experiment_json = experiment_utils._populate_experiment( name, 'mirrored', 'DISTRIBUTED_TRAINING', None, description, app_id, None, None) experiment_json = experiment_utils._attach_experiment_xattr( app_id, run_id, experiment_json, 'CREATE') logdir, return_dict = mirrored_impl._run(sc, train_fn, run_id, local_logdir=local_logdir, name=name, evaluator=evaluator) duration = experiment_utils._seconds_to_milliseconds(time.time() - start) metric = experiment_utils._get_metric(return_dict, metric_key) experiment_utils._finalize_experiment(experiment_json, metric, app_id, run_id, 'FINISHED', duration, logdir, None, None) return logdir, return_dict except: _exception_handler( experiment_utils._seconds_to_milliseconds(time.time() - start)) raise finally: _end_run(sc)
def grid_search(map_fun, args_dict, direction='max', name='no-name', local_logdir=False, versioned_resources=None, description=None): """ *Parallel Experiment* Run multiple experiments and test a grid of hyperparameters for a neural network to maximize e.g. a Neural Network's accuracy. The following example will run *train_nn* with 6 different hyperparameter combinations >>> from hops import experiment >>> grid_dict = {'learning_rate':[0.1, 0.3], 'dropout': [0.4, 0.6, 0.1]} >>> def train_nn(learning_rate, dropout): >>> import tensorflow >>> # code for preprocessing, training and exporting model >>> # mandatory return a value for the experiment which is registered in Experiments service >>> return network.evaluate(learning_rate, dropout) >>> experiment.grid_search(train_nn, grid_dict, direction='max') The following values will be injected in the function and run and evaluated. - (learning_rate=0.1, dropout=0.4) - (learning_rate=0.1, dropout=0.6) - (learning_rate=0.1, dropout=0.1) - (learning_rate=0.3, dropout=0.4) - (learning_rate=0.3, dropout=0.6) - (learning_rate=0.3, dropout=0.1) Args: :map_fun: the function to run, must return a metric :args_dict: a dict with a key for each argument with a corresponding value being a list containing the hyperparameters to test, internally all possible combinations will be generated and run as separate Experiments :direction: 'max' to maximize the returned metric, 'min' to minize the returned metric :name: name of the experiment :local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS :versioned_resources: A list of HDFS paths of resources to version with this experiment :description: a longer description for the experiment Returns: HDFS path in your project where the experiment is stored """ num_ps = util.num_param_servers() assert num_ps == 0, "number of parameter servers should be 0" global running if running: raise RuntimeError("An experiment is currently running. Please call experiment.end() to stop it.") try: global app_id global experiment_json global elastic_id running = True sc = util._find_spark().sparkContext app_id = str(sc.applicationId) gs.run_id = gs.run_id + 1 versioned_path = util._version_resources(versioned_resources, gs._get_logdir(app_id)) experiment_json = util._populate_experiment(sc, name, 'experiment', 'grid_search', gs._get_logdir(app_id), json.dumps(args_dict), versioned_path, description) util._version_resources(versioned_resources, gs._get_logdir(app_id)) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) grid_params = util.grid_params(args_dict) tensorboard_logdir, param, metric = gs._grid_launch(sc, map_fun, grid_params, direction=direction, local_logdir=local_logdir, name=name) experiment_json = util._finalize_experiment(experiment_json, param, metric) util._put_elastic(hopshdfs.project_name(), app_id, elastic_id, experiment_json) except: _exception_handler() raise finally: #cleanup spark jobs elastic_id +=1 running = False sc.setJobGroup("", "") return tensorboard_logdir