def _wrapper_fun(iter): """ Args: iter: Returns: """ for i in iter: executor_num = i experiment_utils._set_ml_id(app_id, run_id) tb_hdfs_path = '' hdfs_exec_logdir = '' t = threading.Thread(target=devices._print_periodic_gpu_utilization) if devices.get_num_gpus() > 0: t.start() try: #Arguments if args_dict: param_string, params, args = experiment_utils.build_parameters( map_fun, executor_num, args_dict) hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories( app_id, run_id, param_string, 'random_search', params=params) logfile = experiment_utils._init_logger(hdfs_exec_logdir) tb_hdfs_path, tb_pid = tensorboard._register( hdfs_exec_logdir, hdfs_appid_logdir, executor_num, local_logdir=local_logdir) print(devices._get_gpu_info()) print( '-------------------------------------------------------') print('Started running task ' + param_string) task_start = time.time() retval = map_fun(*args) task_end = time.time() experiment_utils._handle_return(retval, hdfs_exec_logdir, optimization_key, logfile) time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff( task_start, task_end) print(time_str) print('Returning metric ' + str(retval)) print( '-------------------------------------------------------') except: raise finally: experiment_utils._cleanup(tensorboard, t)
def _wrapper_fun(iter): """ Args: :iter: Returns: """ for i in iter: executor_num = i experiment_utils._set_ml_id(app_id, run_id) tb_hdfs_path = '' hdfs_exec_logdir = '' t = threading.Thread(target=devices._print_periodic_gpu_utilization) if devices.get_num_gpus() > 0: t.start() global local_logdir_bool try: #Arguments if args_dict: param_string, params, args = experiment_utils.build_parameters(map_fun, executor_num, args_dict) val = _get_return_file(param_string, app_id, generation_id, run_id) hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories(app_id, run_id, param_string, 'differential_evolution', sub_type='generation.' + str(generation_id), params=params) logfile = experiment_utils._init_logger(hdfs_exec_logdir) tb_hdfs_path, tb_pid = tensorboard._register(hdfs_exec_logdir, hdfs_appid_logdir, executor_num, local_logdir=local_logdir_bool) print(devices._get_gpu_info()) print('-------------------------------------------------------') print('Started running task ' + param_string) if val is not None: val = json.loads(val) task_start = time.time() if val is None: val = map_fun(*args) task_end = time.time() time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff(task_start, task_end) print(time_str) experiment_utils._handle_return(val, hdfs_exec_logdir, opt_key, logfile) print('Returning metric ' + str(val)) print('-------------------------------------------------------') except: raise finally: experiment_utils._cleanup(tensorboard, t)
def _wrapper_fun(iter): """ Args: iter: Returns: """ for i in iter: executor_num = i experiment_utils._set_ml_id(app_id, run_id) t = threading.Thread(target=devices._print_periodic_gpu_utilization) if devices.get_num_gpus() > 0: t.start() is_chief = False logdir = None tb_hdfs_path = None try: host = experiment_utils._get_ip_address() tmp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tmp_socket.bind(('', 0)) port = tmp_socket.getsockname()[1] client = allreduce_reservation.Client(server_addr) host_port = host + ":" + str(port) client.register({"worker": host_port, "index": executor_num}) cluster = client.await_reservations() tmp_socket.close() client.close() task_index = experiment_utils._find_index(host_port, cluster) if task_index == -1: cluster["task"] = {"type": "chief", "index": 0} else: cluster["task"] = {"type": "worker", "index": task_index} evaluator_node = None if evaluator: last_worker_index = len(cluster["cluster"]["worker"]) - 1 evaluator_node = cluster["cluster"]["worker"][ last_worker_index] cluster["cluster"]["evaluator"] = [evaluator_node] del cluster["cluster"]["worker"][last_worker_index] if evaluator_node == host_port: cluster["task"] = {"type": "evaluator", "index": 0} print('TF_CONFIG: {} '.format(cluster)) if num_executors > 1: os.environ["TF_CONFIG"] = json.dumps(cluster) is_chief = (cluster["task"]["type"] == "chief") is_evaluator = (cluster["task"]["type"] == "evaluator") if is_chief: logdir = experiment_utils._get_logdir(app_id, run_id) tb_hdfs_path, tb_pid = tensorboard._register( logdir, logdir, executor_num, local_logdir=local_logdir) elif is_evaluator: logdir = experiment_utils._get_logdir(app_id, run_id) tensorboard.events_logdir = logdir logfile = experiment_utils._init_logger( experiment_utils._get_logdir(app_id, run_id), role=cluster["task"]["type"], index=cluster["task"]["index"]) print(devices._get_gpu_info()) print('-------------------------------------------------------') print('Started running task') task_start = time.time() retval = map_fun() if is_chief: experiment_utils._handle_return_simple( retval, experiment_utils._get_logdir(app_id, run_id), logfile) task_end = time.time() time_str = 'Finished task - took ' + experiment_utils._time_diff( task_start, task_end) print(time_str) print('-------------------------------------------------------') except: raise finally: experiment_utils._cleanup(tensorboard, t)
def finalize(self, job_end): results = "" if self.experiment_type == "optimization": _ = self.optimizer.finalize_experiment(self._final_store) self.job_end = job_end self.duration = experiment_utils._seconds_to_milliseconds( self.job_end - self.job_start) self.duration_str = experiment_utils._time_diff( self.job_start, self.job_end) results = ("\n------ " + self.optimizer.name() + " Results ------ direction(" + self.direction + ") \n" "BEST combination " + json.dumps(self.result["best_hp"]) + " -- metric " + str(self.result["best_val"]) + "\n" "WORST combination " + json.dumps(self.result["worst_hp"]) + " -- metric " + str(self.result["worst_val"]) + "\n" "AVERAGE metric -- " + str(self.result["avg"]) + "\n" "EARLY STOPPED Trials -- " + str(self.result["early_stopped"]) + "\n" "Total job time " + self.duration_str + "\n") elif self.experiment_type == "ablation": _ = self.ablator.finalize_experiment(self._final_store) self.job_end = job_end self.duration = experiment_utils._seconds_to_milliseconds( self.job_end - self.job_start) self.duration_str = experiment_utils._time_diff( self.job_start, self.job_end) results = ("\n------ " + self.ablator.name() + " Results ------ \n" + "BEST Config Excludes " + json.dumps(self.result["best_config"]) + " -- metric " + str(self.result["best_val"]) + "\n" + "WORST Config Excludes " + json.dumps(self.result["worst_config"]) + " -- metric " + str(self.result["worst_val"]) + "\n" + "AVERAGE metric -- " + str(self.result["avg"]) + "\n" + "Total Job Time " + self.duration_str + "\n") print(results) self._log(results) hopshdfs.dump( json.dumps(self.result, default=util.json_default_numpy), self.log_dir + "/result.json", ) sc = hopsutil._find_spark().sparkContext hopshdfs.dump(self.json(sc), self.log_dir + "/maggy.json") return self.result
def _wrapper_fun(iter): """ Args: iter: Returns: """ for i in iter: executor_num = i experiment_utils._set_ml_id(app_id, run_id) t = threading.Thread(target=devices._print_periodic_gpu_utilization) if devices.get_num_gpus() > 0: t.start() role = None logdir = None tb_hdfs_path = None client = parameter_server_reservation.Client(server_addr) try: host = experiment_utils._get_ip_address() tmp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tmp_socket.bind(('', 0)) port = tmp_socket.getsockname()[1] host_port = host + ":" + str(port) exec_spec = {} if executor_num < num_ps: exec_spec["task_type"] = "ps" else: exec_spec["task_type"] = "worker" exec_spec["host_port"] = host_port exec_spec["gpus_present"] = devices.get_num_gpus() > 0 client.register(exec_spec) cluster = client.await_reservations() tmp_socket.close() role, index = experiment_utils._find_task_and_index(host_port, cluster) cluster_spec = {} cluster_spec["cluster"] = cluster cluster_spec["task"] = {"type": role, "index": index} evaluator_node = None if evaluator: last_worker_index = len(cluster_spec["cluster"]["worker"])-1 evaluator_node = cluster_spec["cluster"]["worker"][last_worker_index] cluster_spec["cluster"]["evaluator"] = [evaluator_node] del cluster_spec["cluster"]["worker"][last_worker_index] if evaluator_node == host_port: role = "evaluator" cluster_spec["task"] = {"type": "evaluator", "index": 0} print('TF_CONFIG: {} '.format(cluster_spec)) os.environ["TF_CONFIG"] = json.dumps(cluster_spec) logfile = experiment_utils._init_logger(experiment_utils._get_logdir(app_id, run_id), role=role, index=cluster_spec["task"]["index"]) dist_logdir = experiment_utils._get_logdir(app_id, run_id) + '/logdir' is_chief = (cluster["task"]["type"] == "chief") if is_chief: hdfs.mkdir(dist_logdir) tensorboard._register(dist_logdir, experiment_utils._get_logdir(app_id, run_id), executor_num, local_logdir=local_logdir) else: tensorboard.events_logdir = dist_logdir print(devices._get_gpu_info()) print('-------------------------------------------------------') print('Started running task') task_start = time.time() retval=None if role == "ps": ps_thread = threading.Thread(target=lambda: map_fun()) ps_thread.start() client.await_all_workers_finished() else: retval = map_fun() if role == "chief": experiment_utils._handle_return_simple(retval, experiment_utils._get_logdir(app_id, run_id), logfile) task_end = time.time() time_str = 'Finished task - took ' + experiment_utils._time_diff(task_start, task_end) print(time_str) print('-------------------------------------------------------') except: raise finally: if role != "ps": client.register_worker_finished() client.close() experiment_utils._cleanup(tensorboard, t)