def __getitem__(self, i): if self.grid_generated == False: log.error( "Grid is not generated, please call .generate first. Exiting ...!", error=True) else: return self.grid_item(i)
def generate_grid(self): grid_list = [] try: for key in list(self.grid_parameters.keys()): grid_list.append(self.grid_parameters[key]) except: log.error( "Grid cannot be generated, check your parameters. Exiting ...!", error=True) self.grid = list(self.grid_operator(*grid_list)) self.grid_generated = True def create_grid_hash__(): total_commands_str = [] #TODO: This can be done better - just need the list itself and not combinations for i in range(len(self.grid)): grid_args = self.gen_args(i) final_command = grid_args total_commands_str.append(final_command) self.grid_hash = get_hash("".join(total_commands_str)) create_grid_hash__() self.grid_dir = os.path.join(self.grid_root, self.grid_hash) if os.path.isdir(self.grid_dir) is True: log.error("Identical grid already exists. Exiting ...!", error=True) log.success("Grid successfully generated.")
def compile_argparse(self, dict_style=False): """ Compiles the parameters using argparse method # Arguments #Returns An argparse object. """ if len(list(self.parameters.keys())) == 0: log.error("No parameters registered. Exiting ...!", error=True) parser = argparse.ArgumentParser() for key in list(self.parameters.keys()): choices = None if self.parameters[key][ 3] == [] else self.parameters[key][3] parser.add_argument(str(key), type=self.parameters[key][0], default=self.parameters[key][1], help=self.parameters[key][2], choices=choices) if dict_style is False: return parser.parse_args() else: return vars(parser.parse_args())
def json_interpret(self, main_res_file, output_file): started, finished, failed, not_started = self.get_status() if len(finished) == 0: log.error("No results to compile yet. Exiting ...!", error=True) import json res_list = [] for command_hex in finished: main_res_fpath = os.path.join(self.grid_dir, "instances/", command_hex + "/", main_res_file) if not os.path.isfile(main_res_fpath): log.error("%s does not have the %s file." % (command_hex, main_res_file), error=False) continue main_res_content = open(main_res_fpath, "r").read() main_res = json.loads(main_res_content) in_params_fpath = os.path.join(self.grid_dir, "instances/", command_hex + "/", command_hex + ".pkl") in_params = pickle.load(open(in_params_fpath, "rb")) for key in in_params: main_res["STDGRID_%s" % key] = in_params[key] res_list.append(main_res) res_list[-1]["STDGRID_command_hex"] = command_hex res_list[-1]["STDGRID_grid_hex"] = os.path.basename(self.grid_dir) self.__write_res_to_csv(res_list, output_file)
def grid_item(self,i): if self.grid_generated==False: log.error("Grid is not generated, cannot get item. Exiting ...!",error=True) output={} for key,j in zip(list(self.grid_parameters.keys()),range(len(list(self.grid_parameters.keys())))): output[key]=self.grid[i][j] return output
def register_parameter(self, __name, __type, __default, __help=""): """ Registers a parameter for machine learning code # Arguments __name: The name of the variable to be registered. __type: The type of the variable. __default: Default value to be passed to the variable. __help: Helper for the parameter. #Returns """ if __name in list(self.parameters.keys()): log.error("Parameter already registered. Exiting ...!", error=True) self.parameters[__name] = [__type, __default, __help]
def generate_shell_instances(self, prefix="", postfix=""): if self.shell_instances_generated: log.error("Shell instances already generated. Exiting ...!") os.makedirs(self.grid_dir) instances_dir = os.path.join(self.grid_dir, "instances/") entry_point_relative_to_instance = os.path.join( "../../../", self.entry) def write_shell_instance_content__(fhandle, command, command_hex): fhandle.write("#!/bin/sh\n") fhandle.write("run_grid_instance (){\n") fhandle.write("echo \"STARTED\" > shell_instance_started_signal\n") fhandle.write("\t" + command + " \n ") fhandle.write("\t%s > %s\n" % ("echo $?", "STANDARDGRID_instance_output")) fhandle.write("rm shell_instance_started_signal\n") fhandle.write("}\n") fhandle.write("run_grid_instance") def write_instance_parameters__(fhandle, grid_instance): pickle.dump(grid_instance, fhandle) for i in range(len(self.grid)): grid_instance = self.gen_args(i) command = prefix + " " + entry_point_relative_to_instance + " " + grid_instance command_hex = get_hash(command) command = command + " " + postfix command_dir = os.path.join(instances_dir, command_hex) os.makedirs(command_dir) local_sh_name = os.path.join(command_dir, command_hex + ".sh") write_shell_instance_content__(open(local_sh_name, "w"), command, command_hex) instance_pkl_fname = os.path.join(command_dir, command_hex + ".pkl") write_instance_parameters__(open(instance_pkl_fname, "wb"), self[i]) self.shell_instances_generated = True log.success("Shell instances created for grid in %s/instances" % self.grid_dir)
def apply(self,apply_fn,input_file,output_file): started,finished,failed,not_started=self.get_status() if len(finished)==0: log.error("No results to compile yet. Exiting ...!",error=True) for command_hex in finished: in_fpath=os.path.join(self.grid_dir,"instances/",command_hex+"/",input_file) out_fpath=os.path.join(self.grid_dir,"instances/",command_hex+"/",output_file) if not os.path.isfile(in_fpath): log.warning("%s does not have the %s file."%(command_hex,input_file)) continue try: apply_fn(in_fpath,out_fpath) except Exception as e: log.error("Cannot apply the given function to instance %s with error %s ...!"%(command_hex,str(e)),error=False) log.success("Function application finished.")
def __init__(self, entry_script_, grid_root_, resume_on_retry_=True, grid_operator_=product, description_="Grid search object"): if os.path.isfile(entry_script_) is False: log.error("Entry point does not exist. Exiting ...!", error=True) self.entry_script = entry_script_ self.entry = os.path.relpath(self.entry_script, grid_root_) self.grid_root = os.path.relpath( grid_root_, os.sep.join(sys.argv[0].split(os.sep)[:-1])) if os.path.isdir(self.grid_root) is False: os.makedirs(self.grid_root) self.resume_on_retry = resume_on_retry_ self.grid_operator = grid_operator_ self.description = description_ self.grid_parameters = OrderedDict() self.shell_instances_generated = False self.grid_generated = False self.grid_saved = False
def __init__(self,entry_script_, grid_root_,hash_len=4, email_args=None, resume_on_retry_=True,grid_operator_=product,description_="Grid search object"): '''hash_len: number of characters to use in hashes''' if os.path.isfile(entry_script_) is False: log.error("Entry point does not exist. Exiting ...!",error=True) self.entry_script=entry_script_ self.entry=os.path.relpath(self.entry_script,grid_root_) self.grid_root=os.path.relpath(grid_root_,os.sep.join(sys.argv[0].split(os.sep)[:-1])) if os.path.isdir(self.grid_root) is False: os.makedirs(self.grid_root) self.resume_on_retry=resume_on_retry_ self.grid_operator=grid_operator_ self.description=description_ self.grid_parameters=OrderedDict() self.shell_instances_generated=False self.grid_generated=False self.grid_saved=False self.rt=Runtime() self.hash_len=hash_len self.email_sent=False self.email_args = email_args self.final_status=None
def register(self,key,value): if self.grid_generated==True: log.error("Grid already generated, cannot register any more hypers. Exiting ...!",error=True) if type(key)!=str: log.error("Key to be registered is not a string. Exiting ...!",error=True) if type(value)!=list: log.error("Value to be registered is not a list. Exiting ...!",error=True) self.grid_parameters[key]=value log.status("%s registered in grid."%key)
def shuffle_grid(self): if self.grid_generated == False: log.error("Grid was not generated, cannot shuffle. Exiting ...!", error=True) random.shuffle(self.grid)
def create_runner(self, fraction=1.0, num_runners=1, runners_prefix=["sh"], parallel=1, hard_resume=False): self.__write_description() if parallel > 3: log.error("Parallel cannot be higher than 3.", error=True) if fraction > 1 or fraction < 0: log.error("Fraction not in range [0,1].", error=True) if num_runners == None: num_runners = len(self.grid) if len(runners_prefix) == 1: runners_prefix = runners_prefix * num_runners if len(runners_prefix) != num_runners: log.error( "Mismatch between num_runners and runners_prefix arguments. Exiting ...!", error=True) last_run_params = { "fraction": fraction, "num_runners": num_runners, "runners_prefix": runners_prefix, "parallel": parallel } self.last_run_params = last_run_params if hard_resume: while True: permission = log.status( "Specified hard_resume, are you sure you want to add the unfinished instances to the grid? (y,n)", require_input=True) if permission == "y": break elif permission == "n": exit() else: pass started, finished, failed, not_started = self.get_status() if hard_resume: command_hexes = started + failed + not_started else: command_hexes = failed + not_started if len(command_hexes) == 0: log.advisory("No more instances remaining. Exiting ...!") exit() #Ensure no script output is recorded, therefore all the instances will be seen as not-started. self.__nullify_previous_instance_runs(command_hexes) #If the central command directory does not exist, then recreate it central_command_dir = os.path.join(self.grid_dir, "central/") if not os.path.exists(central_command_dir): os.makedirs(central_command_dir) temp_counter = 0 while (True): attempt = "attempt_%d/" % temp_counter attempt_dir = os.path.join(self.grid_dir, "central/", attempt) if os.path.exists(attempt_dir): temp_counter += 1 else: break os.makedirs(attempt_dir) group_dir = os.path.join(self.grid_dir, "central/", attempt, "groups/") if not os.path.exists(group_dir): os.makedirs(group_dir) main_handle = open(os.path.join(attempt_dir, "main.sh"), "w") def write_main_entries__(main_handle, entry): main_handle.write("cd ./groups/\n") main_handle.write("%s\n" % entry) main_handle.write("cd - > /dev/null\n") split_len = math.ceil((len(self.grid) * fraction) / num_runners) run_counter = 0 for i in range(num_runners): this_group = "group_%d.sh" % i this_hexes = command_hexes[i * split_len:(i + 1) * split_len] this_group_fname = os.path.join(group_dir, this_group) group_handle = open(this_group_fname, "w") for this_hex in this_hexes: group_handle.write( "cd ../../../instances/%s/ && echo Running %s && %s %s.sh > %s.stdout && cd - > /dev/null\n" % (this_hex, this_hex, runners_prefix[i], this_hex, this_hex)) write_main_entries__( main_handle, "cat %s | xargs -L 1 -I CMD -P %d bash -c CMD &" % (this_group, parallel)) main_handle.write("wait") main_handle.close() log.success("Grid runners established under %s" % os.path.join(self.grid_dir, "central", attempt))