def generate_grid(self):
grid_list = []
try:
for key in list(self.grid_parameters.keys()):
grid_list.append(self.grid_parameters[key])
except:
log.error(
"Grid cannot be generated, check your parameters. Exiting ...!",
error=True)
self.grid = list(self.grid_operator(*grid_list))
self.grid_generated = True
def create_grid_hash__():
total_commands_str = []
#TODO: This can be done better - just need the list itself and not combinations
for i in range(len(self.grid)):
grid_args = self.gen_args(i)
final_command = grid_args
total_commands_str.append(final_command)
self.grid_hash = get_hash("".join(total_commands_str))
create_grid_hash__()
self.grid_dir = os.path.join(self.grid_root, self.grid_hash)
if os.path.isdir(self.grid_dir) is True:
log.error("Identical grid already exists. Exiting ...!",
error=True)
log.success("Grid successfully generated.")
def apply(self, apply_fn, input_file, output_file):
started, finished, failed, not_started = self.get_status()
if len(finished) == 0:
log.error("No results to compile yet. Exiting ...!", error=True)
for command_hex in finished:
in_fpath = os.path.join(self.grid_dir, "instances/",
command_hex + "/", input_file)
out_fpath = os.path.join(self.grid_dir, "instances/",
command_hex + "/", output_file)
if not os.path.isfile(in_fpath):
log.error("%s does not have the %s file." %
(command_hex, input_file),
error=False)
continue
try:
apply_fn(in_fpath, out_fpath)
except Exception as e:
log.error(
"Cannot apply the given function to instance %s with error %s ...!"
% (command_hex, str(e)),
error=False)
log.success("Function application finished.")
def generate_shell_instances(self, prefix="", postfix=""):
if self.shell_instances_generated:
log.error("Shell instances already generated. Exiting ...!")
os.makedirs(self.grid_dir)
instances_dir = os.path.join(self.grid_dir, "instances/")
entry_point_relative_to_instance = os.path.join(
"../../../", self.entry)
def write_shell_instance_content__(fhandle, command, command_hex):
fhandle.write("#!/bin/sh\n")
fhandle.write("run_grid_instance (){\n")
fhandle.write("echo \"STARTED\" > shell_instance_started_signal\n")
fhandle.write("\t" + command + " \n ")
fhandle.write("\t%s > %s\n" %
("echo $?", "STANDARDGRID_instance_output"))
fhandle.write("rm shell_instance_started_signal\n")
fhandle.write("}\n")
fhandle.write("run_grid_instance")
def write_instance_parameters__(fhandle, grid_instance):
pickle.dump(grid_instance, fhandle)
for i in range(len(self.grid)):
grid_instance = self.gen_args(i)
command = prefix + " " + entry_point_relative_to_instance + " " + grid_instance
command_hex = get_hash(command)
command = command + " " + postfix
command_dir = os.path.join(instances_dir, command_hex)
os.makedirs(command_dir)
local_sh_name = os.path.join(command_dir, command_hex + ".sh")
write_shell_instance_content__(open(local_sh_name, "w"), command,
command_hex)
instance_pkl_fname = os.path.join(command_dir,
command_hex + ".pkl")
write_instance_parameters__(open(instance_pkl_fname, "wb"),
self[i])
self.shell_instances_generated = True
log.success("Shell instances created for grid in %s/instances" %
self.grid_dir)
def __write_res_to_csv(self, res_list, output_fname):
import csv
with open(output_fname, "w") as output:
csv_writer = csv.writer(output,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
#Writing the headers first
res0 = res_list[0]
header = []
for key in res0:
header.append(key)
csv_writer.writerow(header)
for res in res_list:
row = []
for key in res0:
row.append(res[key])
csv_writer.writerow(row)
log.success("Results gathered in %s" % output_fname)
def create_runner(self,
fraction=1.0,
num_runners=1,
runners_prefix=["sh"],
parallel=1,
hard_resume=False):
self.__write_description()
if parallel > 3:
log.error("Parallel cannot be higher than 3.", error=True)
if fraction > 1 or fraction < 0:
log.error("Fraction not in range [0,1].", error=True)
if num_runners == None:
num_runners = len(self.grid)
if len(runners_prefix) == 1:
runners_prefix = runners_prefix * num_runners
if len(runners_prefix) != num_runners:
log.error(
"Mismatch between num_runners and runners_prefix arguments. Exiting ...!",
error=True)
last_run_params = {
"fraction": fraction,
"num_runners": num_runners,
"runners_prefix": runners_prefix,
"parallel": parallel
}
self.last_run_params = last_run_params
if hard_resume:
while True:
permission = log.status(
"Specified hard_resume, are you sure you want to add the unfinished instances to the grid? (y,n)",
require_input=True)
if permission == "y":
break
elif permission == "n":
exit()
else:
pass
started, finished, failed, not_started = self.get_status()
if hard_resume:
command_hexes = started + failed + not_started
else:
command_hexes = failed + not_started
if len(command_hexes) == 0:
log.advisory("No more instances remaining. Exiting ...!")
exit()
#Ensure no script output is recorded, therefore all the instances will be seen as not-started.
self.__nullify_previous_instance_runs(command_hexes)
#If the central command directory does not exist, then recreate it
central_command_dir = os.path.join(self.grid_dir, "central/")
if not os.path.exists(central_command_dir):
os.makedirs(central_command_dir)
temp_counter = 0
while (True):
attempt = "attempt_%d/" % temp_counter
attempt_dir = os.path.join(self.grid_dir, "central/", attempt)
if os.path.exists(attempt_dir):
temp_counter += 1
else:
break
os.makedirs(attempt_dir)
group_dir = os.path.join(self.grid_dir, "central/", attempt, "groups/")
if not os.path.exists(group_dir):
os.makedirs(group_dir)
main_handle = open(os.path.join(attempt_dir, "main.sh"), "w")
def write_main_entries__(main_handle, entry):
main_handle.write("cd ./groups/\n")
main_handle.write("%s\n" % entry)
main_handle.write("cd - > /dev/null\n")
split_len = math.ceil((len(self.grid) * fraction) / num_runners)
run_counter = 0
for i in range(num_runners):
this_group = "group_%d.sh" % i
this_hexes = command_hexes[i * split_len:(i + 1) * split_len]
this_group_fname = os.path.join(group_dir, this_group)
group_handle = open(this_group_fname, "w")
for this_hex in this_hexes:
group_handle.write(
"cd ../../../instances/%s/ && echo Running %s && %s %s.sh > %s.stdout && cd - > /dev/null\n"
% (this_hex, this_hex, runners_prefix[i], this_hex,
this_hex))
write_main_entries__(
main_handle, "cat %s | xargs -L 1 -I CMD -P %d bash -c CMD &" %
(this_group, parallel))
main_handle.write("wait")
main_handle.close()
log.success("Grid runners established under %s" %
os.path.join(self.grid_dir, "central", attempt))