def __getitem__(self, i):
if self.grid_generated == False:
log.error(
"Grid is not generated, please call .generate first. Exiting ...!",
error=True)
else:
return self.grid_item(i)
def generate_grid(self):
grid_list = []
try:
for key in list(self.grid_parameters.keys()):
grid_list.append(self.grid_parameters[key])
except:
log.error(
"Grid cannot be generated, check your parameters. Exiting ...!",
error=True)
self.grid = list(self.grid_operator(*grid_list))
self.grid_generated = True
def create_grid_hash__():
total_commands_str = []
#TODO: This can be done better - just need the list itself and not combinations
for i in range(len(self.grid)):
grid_args = self.gen_args(i)
final_command = grid_args
total_commands_str.append(final_command)
self.grid_hash = get_hash("".join(total_commands_str))
create_grid_hash__()
self.grid_dir = os.path.join(self.grid_root, self.grid_hash)
if os.path.isdir(self.grid_dir) is True:
log.error("Identical grid already exists. Exiting ...!",
error=True)
log.success("Grid successfully generated.")
def compile_argparse(self, dict_style=False):
""" Compiles the parameters using argparse method
# Arguments
#Returns
An argparse object.
"""
if len(list(self.parameters.keys())) == 0:
log.error("No parameters registered. Exiting ...!", error=True)
parser = argparse.ArgumentParser()
for key in list(self.parameters.keys()):
choices = None if self.parameters[key][
3] == [] else self.parameters[key][3]
parser.add_argument(str(key),
type=self.parameters[key][0],
default=self.parameters[key][1],
help=self.parameters[key][2],
choices=choices)
if dict_style is False:
return parser.parse_args()
else:
return vars(parser.parse_args())
def json_interpret(self, main_res_file, output_file):
started, finished, failed, not_started = self.get_status()
if len(finished) == 0:
log.error("No results to compile yet. Exiting ...!", error=True)
import json
res_list = []
for command_hex in finished:
main_res_fpath = os.path.join(self.grid_dir, "instances/",
command_hex + "/", main_res_file)
if not os.path.isfile(main_res_fpath):
log.error("%s does not have the %s file." %
(command_hex, main_res_file),
error=False)
continue
main_res_content = open(main_res_fpath, "r").read()
main_res = json.loads(main_res_content)
in_params_fpath = os.path.join(self.grid_dir, "instances/",
command_hex + "/",
command_hex + ".pkl")
in_params = pickle.load(open(in_params_fpath, "rb"))
for key in in_params:
main_res["STDGRID_%s" % key] = in_params[key]
res_list.append(main_res)
res_list[-1]["STDGRID_command_hex"] = command_hex
res_list[-1]["STDGRID_grid_hex"] = os.path.basename(self.grid_dir)
self.__write_res_to_csv(res_list, output_file)
def grid_item(self,i):
if self.grid_generated==False:
log.error("Grid is not generated, cannot get item. Exiting ...!",error=True)
output={}
for key,j in zip(list(self.grid_parameters.keys()),range(len(list(self.grid_parameters.keys())))):
output[key]=self.grid[i][j]
return output
def register_parameter(self, __name, __type, __default, __help=""):
""" Registers a parameter for machine learning code
# Arguments
__name: The name of the variable to be registered.
__type: The type of the variable.
__default: Default value to be passed to the variable.
__help: Helper for the parameter.
#Returns
"""
if __name in list(self.parameters.keys()):
log.error("Parameter already registered. Exiting ...!", error=True)
self.parameters[__name] = [__type, __default, __help]
def generate_shell_instances(self, prefix="", postfix=""):
if self.shell_instances_generated:
log.error("Shell instances already generated. Exiting ...!")
os.makedirs(self.grid_dir)
instances_dir = os.path.join(self.grid_dir, "instances/")
entry_point_relative_to_instance = os.path.join(
"../../../", self.entry)
def write_shell_instance_content__(fhandle, command, command_hex):
fhandle.write("#!/bin/sh\n")
fhandle.write("run_grid_instance (){\n")
fhandle.write("echo \"STARTED\" > shell_instance_started_signal\n")
fhandle.write("\t" + command + " \n ")
fhandle.write("\t%s > %s\n" %
("echo $?", "STANDARDGRID_instance_output"))
fhandle.write("rm shell_instance_started_signal\n")
fhandle.write("}\n")
fhandle.write("run_grid_instance")
def write_instance_parameters__(fhandle, grid_instance):
pickle.dump(grid_instance, fhandle)
for i in range(len(self.grid)):
grid_instance = self.gen_args(i)
command = prefix + " " + entry_point_relative_to_instance + " " + grid_instance
command_hex = get_hash(command)
command = command + " " + postfix
command_dir = os.path.join(instances_dir, command_hex)
os.makedirs(command_dir)
local_sh_name = os.path.join(command_dir, command_hex + ".sh")
write_shell_instance_content__(open(local_sh_name, "w"), command,
command_hex)
instance_pkl_fname = os.path.join(command_dir,
command_hex + ".pkl")
write_instance_parameters__(open(instance_pkl_fname, "wb"),
self[i])
self.shell_instances_generated = True
log.success("Shell instances created for grid in %s/instances" %
self.grid_dir)
def apply(self,apply_fn,input_file,output_file):
started,finished,failed,not_started=self.get_status()
if len(finished)==0:
log.error("No results to compile yet. Exiting ...!",error=True)
for command_hex in finished:
in_fpath=os.path.join(self.grid_dir,"instances/",command_hex+"/",input_file)
out_fpath=os.path.join(self.grid_dir,"instances/",command_hex+"/",output_file)
if not os.path.isfile(in_fpath):
log.warning("%s does not have the %s file."%(command_hex,input_file))
continue
try:
apply_fn(in_fpath,out_fpath)
except Exception as e:
log.error("Cannot apply the given function to instance %s with error %s ...!"%(command_hex,str(e)),error=False)
log.success("Function application finished.")
def __init__(self,
entry_script_,
grid_root_,
resume_on_retry_=True,
grid_operator_=product,
description_="Grid search object"):
if os.path.isfile(entry_script_) is False:
log.error("Entry point does not exist. Exiting ...!", error=True)
self.entry_script = entry_script_
self.entry = os.path.relpath(self.entry_script, grid_root_)
self.grid_root = os.path.relpath(
grid_root_, os.sep.join(sys.argv[0].split(os.sep)[:-1]))
if os.path.isdir(self.grid_root) is False:
os.makedirs(self.grid_root)
self.resume_on_retry = resume_on_retry_
self.grid_operator = grid_operator_
self.description = description_
self.grid_parameters = OrderedDict()
self.shell_instances_generated = False
self.grid_generated = False
self.grid_saved = False
def __init__(self,entry_script_, grid_root_,hash_len=4, email_args=None, resume_on_retry_=True,grid_operator_=product,description_="Grid search object"):
'''hash_len: number of characters to use in hashes'''
if os.path.isfile(entry_script_) is False:
log.error("Entry point does not exist. Exiting ...!",error=True)
self.entry_script=entry_script_
self.entry=os.path.relpath(self.entry_script,grid_root_)
self.grid_root=os.path.relpath(grid_root_,os.sep.join(sys.argv[0].split(os.sep)[:-1]))
if os.path.isdir(self.grid_root) is False:
os.makedirs(self.grid_root)
self.resume_on_retry=resume_on_retry_
self.grid_operator=grid_operator_
self.description=description_
self.grid_parameters=OrderedDict()
self.shell_instances_generated=False
self.grid_generated=False
self.grid_saved=False
self.rt=Runtime()
self.hash_len=hash_len
self.email_sent=False
self.email_args = email_args
self.final_status=None
def register(self,key,value):
if self.grid_generated==True:
log.error("Grid already generated, cannot register any more hypers. Exiting ...!",error=True)
if type(key)!=str:
log.error("Key to be registered is not a string. Exiting ...!",error=True)
if type(value)!=list:
log.error("Value to be registered is not a list. Exiting ...!",error=True)
self.grid_parameters[key]=value
log.status("%s registered in grid."%key)
def shuffle_grid(self):
if self.grid_generated == False:
log.error("Grid was not generated, cannot shuffle. Exiting ...!",
error=True)
random.shuffle(self.grid)
def create_runner(self,
fraction=1.0,
num_runners=1,
runners_prefix=["sh"],
parallel=1,
hard_resume=False):
self.__write_description()
if parallel > 3:
log.error("Parallel cannot be higher than 3.", error=True)
if fraction > 1 or fraction < 0:
log.error("Fraction not in range [0,1].", error=True)
if num_runners == None:
num_runners = len(self.grid)
if len(runners_prefix) == 1:
runners_prefix = runners_prefix * num_runners
if len(runners_prefix) != num_runners:
log.error(
"Mismatch between num_runners and runners_prefix arguments. Exiting ...!",
error=True)
last_run_params = {
"fraction": fraction,
"num_runners": num_runners,
"runners_prefix": runners_prefix,
"parallel": parallel
}
self.last_run_params = last_run_params
if hard_resume:
while True:
permission = log.status(
"Specified hard_resume, are you sure you want to add the unfinished instances to the grid? (y,n)",
require_input=True)
if permission == "y":
break
elif permission == "n":
exit()
else:
pass
started, finished, failed, not_started = self.get_status()
if hard_resume:
command_hexes = started + failed + not_started
else:
command_hexes = failed + not_started
if len(command_hexes) == 0:
log.advisory("No more instances remaining. Exiting ...!")
exit()
#Ensure no script output is recorded, therefore all the instances will be seen as not-started.
self.__nullify_previous_instance_runs(command_hexes)
#If the central command directory does not exist, then recreate it
central_command_dir = os.path.join(self.grid_dir, "central/")
if not os.path.exists(central_command_dir):
os.makedirs(central_command_dir)
temp_counter = 0
while (True):
attempt = "attempt_%d/" % temp_counter
attempt_dir = os.path.join(self.grid_dir, "central/", attempt)
if os.path.exists(attempt_dir):
temp_counter += 1
else:
break
os.makedirs(attempt_dir)
group_dir = os.path.join(self.grid_dir, "central/", attempt, "groups/")
if not os.path.exists(group_dir):
os.makedirs(group_dir)
main_handle = open(os.path.join(attempt_dir, "main.sh"), "w")
def write_main_entries__(main_handle, entry):
main_handle.write("cd ./groups/\n")
main_handle.write("%s\n" % entry)
main_handle.write("cd - > /dev/null\n")
split_len = math.ceil((len(self.grid) * fraction) / num_runners)
run_counter = 0
for i in range(num_runners):
this_group = "group_%d.sh" % i
this_hexes = command_hexes[i * split_len:(i + 1) * split_len]
this_group_fname = os.path.join(group_dir, this_group)
group_handle = open(this_group_fname, "w")
for this_hex in this_hexes:
group_handle.write(
"cd ../../../instances/%s/ && echo Running %s && %s %s.sh > %s.stdout && cd - > /dev/null\n"
% (this_hex, this_hex, runners_prefix[i], this_hex,
this_hex))
write_main_entries__(
main_handle, "cat %s | xargs -L 1 -I CMD -P %d bash -c CMD &" %
(this_group, parallel))
main_handle.write("wait")
main_handle.close()
log.success("Grid runners established under %s" %
os.path.join(self.grid_dir, "central", attempt))
