path = tempfile.mkdtemp()
w0 = open("%s/part-r-00000" % path, 'w')
w0.write("\t".join(weights) + "\n")
w0.close()
#
# Create schema for weights, place under weight-0 dir
#
w0_schema = {
"fields": w0_fields,
"version": 0,
"sortKeys": [],
"sortKeyOrders": []
}
w0_schema_file = open("%s/.pig_schema" % path, 'w')
ObjectMapper().writeValue(w0_schema_file, w0_schema)
w0_schema_file.close()
#
# Copy initial weights to fs
#
copyFromLocal = "copyFromLocal %s %s/%s" % (path, data_dir, "weight-0")
Pig.fs(copyFromLocal)
#
# Iterate until converged
#
features = "%s/%s" % (data_dir, features)
script = Pig.compileFromFile(pig_script)
weight_queue = Queue.Queue(25) # for moving average
avg_weight = [0.0 for i in xrange(int(num_features))]
