def back_end_process(prefix,base_prefix,faction=""):
"""Do processing specific to a faction or else process a file for "all". """
# This command is needed to, in particular, strip out unneeded values from the RECRUITED column
command = "waffles_transform dropunusedvalues " + sh_escape(prefix) + ".arff"
step1 = strip_quotes.sub('', prefix)
clean_prefix = space_to_underscore.sub("_",step1)
if not faction:
matched = re.match("__[^_]*_(.*)",clean_prefix)
faction = matched.group(1)
print "Now processing the faction: " + str(faction)
temp4_filename = "__temp4_" + faction + ".arff"
do_command(command,True,temp4_filename)
final_prefix = base_prefix + "_" + faction
command = "waffles_transform drophomogcols {0}".format(temp4_filename)
do_command(command,True,final_prefix + ".arff")
command = "waffles_learn train -seed 0 {0}.arff neuralnet {1}".format(final_prefix,options.neural_net_parms)
do_command(command,True,final_prefix + ".json")
sleep(61) # Need to wait one minute to avoid having both jobs write to the same time-stamped log file
all_done = False
while not all_done:
all_done = all(q.poll() != None for q in jobs)
sleep(10)
assert all(not p.poll() for p in jobs),"Error in ai_test2.py!!!! Aborting job."
if not os.path.isdir(model_dir):
os.mkdir(model_dir)
base_file_name = model_dir.strip("/").split("/")[-1]
command = "csv_normalizer.py {0} *.csv ".format(options.old_csv,base_file_name)
normalized_csv = "{0}/{1}.csv".format(model_dir,base_file_name)
do_command(command,True,normalized_csv)
olddir = os.getcwd()
os.chdir(model_dir)
metric = """-m "'{0}'" """.format(options.metric.strip("""'" """)) # Have to strip off, then put back on quotes and spaces
command = "csv_to_model.py {0} {1}".format(base_file_name + ".csv", metric)
do_command(command,True)
os.chdir(olddir)
if options.default_file:
default_file = " -d " + options.default_file
else:
default_file = ""
command = "deploy_models.py {0} {1} {2}".format(default_file,model_dir,deploy_dir)
do_command(command,True)
(options, args) = parser.parse_args()
if (len(args) != 1):
parser.error("Incorrect number of arguments. Requires one .csv file as an argument. Run with --help for help")
input_file = args[0].strip()
current_directory = "."
assert input_file.endswith(".csv"),"Input must be a .csv file!"
assert input_file in os.listdir(current_directory), input_file + " not found in current directory!"
m = re.match("(.*)\.csv",input_file)
file_prefix = m.group(1)
temp0_file = "__temp0.csv"
if temp0_file not in os.listdir(current_directory):
if options.retain_default:
command = "cp " + sh_escape(input_file) + temp0_file
do_command(command,True)
else:
command = "grep -v default " + sh_escape(input_file)
do_command(command,True,temp0_file)
main_arff_file = file_prefix + ".arff"
if main_arff_file not in os.listdir(current_directory):
command = "waffles_transform import {0} -columnnames".format(temp0_file)
do_command(command,True,main_arff_file)
columns = get_schema_from_arff(main_arff_file)
metric = columns.index(options.metric)
command = 'waffles_transform swapcolumns ' + main_arff_file + " " + str(metric) + " " + str(len(columns) -1)
usage = "Turns a normalized .csv file (produced by csv_normalizer) into a .arff file suitable for training\n" + \
"usage: %prog [options] input.arff"
parser = OptionParser(usage)
parser.add_option("-f","--faction",dest="faction",help="Faction you want to learn a model for. Defaults to build a model for all factions.")
parser.add_option("-r","--retain",dest="retain_temporary",help="Retain temporary files?",action="store_true",default=False)
(options, args) = parser.parse_args()
if options.faction == "'Knalgan Alliance'":
faction = Knalgan
else:
faction = options.faction
# Strip out unneeded values from the RECRUITED column
command = "waffles_transform dropunusedvalues " + sh_escape(args[0]) + "_" + sh_escape(args[1])
do_command(command,True,"__temp10_" + faction + ".arff")
command = "waffles_transform drophomogcols __temp10_" + faction + ".arff"
do_command(command,True,"__temp11_" + faction + ".arff")
command = "waffles_learn train -seed 0 __temp11_{0}.arff neuralnet -addlayer {1:d}".format(faction,options.layers)
do_command(command,True,"" "__temp11_" + faction + ".arff")
if not options.retain_temporary:
subprocess.call("rm __temp1* __temp2.arff __temp3.arff __temp4.arff",shell=True)