示例#1
0
def back_end_process(prefix,base_prefix,faction=""):
    """Do processing specific to a faction or else process a file for "all".  """

    # This command is needed to, in particular, strip out unneeded values from the RECRUITED column
    command = "waffles_transform dropunusedvalues " +  sh_escape(prefix) + ".arff"
    step1 = strip_quotes.sub('', prefix)
    clean_prefix = space_to_underscore.sub("_",step1)

    if not faction:
        matched = re.match("__[^_]*_(.*)",clean_prefix)
        faction = matched.group(1)
    print "Now processing the faction: " + str(faction)
    temp4_filename = "__temp4_" + faction +  ".arff"
    do_command(command,True,temp4_filename)

    final_prefix = base_prefix + "_" + faction

    command = "waffles_transform drophomogcols {0}".format(temp4_filename)
    do_command(command,True,final_prefix + ".arff")


    command = "waffles_learn train -seed 0 {0}.arff neuralnet {1}".format(final_prefix,options.neural_net_parms)
    do_command(command,True,final_prefix +  ".json")
    sleep(61)  # Need to wait one minute to avoid having both jobs write to the same time-stamped log file

all_done = False
while not all_done:
    all_done = all(q.poll() != None for q in jobs)
    sleep(10)

assert all(not p.poll() for p in jobs),"Error in ai_test2.py!!!!  Aborting job."

if not os.path.isdir(model_dir):
    os.mkdir(model_dir)

base_file_name = model_dir.strip("/").split("/")[-1]
command = "csv_normalizer.py {0} *.csv ".format(options.old_csv,base_file_name)
normalized_csv = "{0}/{1}.csv".format(model_dir,base_file_name)
do_command(command,True,normalized_csv)
olddir = os.getcwd()
os.chdir(model_dir)

metric = """-m "'{0}'" """.format(options.metric.strip("""'" """))  # Have to strip off, then put back on quotes and spaces
command = "csv_to_model.py {0} {1}".format(base_file_name + ".csv", metric)
do_command(command,True)
os.chdir(olddir)

if options.default_file:
    default_file = " -d " + options.default_file
else:
    default_file = ""

command = "deploy_models.py {0} {1} {2}".format(default_file,model_dir,deploy_dir)
do_command(command,True)
示例#3
0
(options, args) = parser.parse_args()
if (len(args) != 1):
    parser.error("Incorrect number of arguments.  Requires one .csv file as an argument. Run with --help for help")

input_file = args[0].strip()
current_directory = "."
assert input_file.endswith(".csv"),"Input must be a .csv file!"
assert input_file in os.listdir(current_directory), input_file + " not found in current directory!"
m = re.match("(.*)\.csv",input_file)
file_prefix = m.group(1)

temp0_file = "__temp0.csv"
if temp0_file not in os.listdir(current_directory):
    if options.retain_default:
        command = "cp " + sh_escape(input_file) + temp0_file
        do_command(command,True)
    else:
        command = "grep -v default " + sh_escape(input_file)
        do_command(command,True,temp0_file)


main_arff_file = file_prefix + ".arff"
if main_arff_file not in os.listdir(current_directory):
    command = "waffles_transform import {0} -columnnames".format(temp0_file)
    do_command(command,True,main_arff_file)


     
columns = get_schema_from_arff(main_arff_file)
metric = columns.index(options.metric)
command = 'waffles_transform swapcolumns ' + main_arff_file + " " +  str(metric) + " " + str(len(columns) -1)

usage = "Turns a normalized .csv file (produced by csv_normalizer) into a .arff file suitable for training\n" + \
    "usage: %prog [options] input.arff"
parser = OptionParser(usage)
parser.add_option("-f","--faction",dest="faction",help="Faction you want to learn a model for.  Defaults to build a model for all factions.")
parser.add_option("-r","--retain",dest="retain_temporary",help="Retain temporary files?",action="store_true",default=False)
(options, args) = parser.parse_args()

if options.faction == "'Knalgan Alliance'":
    faction = Knalgan
else:
    faction = options.faction

# Strip out unneeded values from the RECRUITED column
command = "waffles_transform dropunusedvalues " +  sh_escape(args[0]) + "_" + sh_escape(args[1])
do_command(command,True,"__temp10_" +  faction +  ".arff")

command = "waffles_transform drophomogcols __temp10_" + faction + ".arff"
do_command(command,True,"__temp11_" + faction + ".arff")

command = "waffles_learn train -seed 0 __temp11_{0}.arff neuralnet -addlayer {1:d}".format(faction,options.layers)
do_command(command,True,""     "__temp11_" + faction + ".arff")


if not options.retain_temporary:
    subprocess.call("rm __temp1* __temp2.arff __temp3.arff __temp4.arff",shell=True)