def startReduction(outdir, email=None):
records = dbq.getAllCompleted(email=email)
if records is None:
print ("Failed to retrieve records from database")
return
print ("Processing {} user records from the database".format(len(records)))
for record in records:
# Reduce the profile and save it out to the fs
data = json_to_object(record.data)
try:
data = rdc.std_reduce(data)
except Exception as e:
print ("Failed to reduce profile with email={}".format(record.email))
print ("Error: " + str(e))
tb = traceback.format_exc()
print tb
sys.exit(2)
data["username"] = record.name
out_file = "{}_reduced.json".format(record.email)
out_file = os.path.join(outdir, out_file)
out_file = out_file.replace(" ", "_").replace("@", "")
print ("wpm:{}".format(data["wpm"]))
print ("name:{} email:{} -> {}".format(record.name, record.email, out_file))
with open(out_file, "w+") as f:
write_json(data, fd=f)
except KeyError as e:
print("Unknown partitioner '{}'".format(args.partitioner))
sys.exit(1)
profiles = loadProfiles(*args.profiles)
(partitions, user_to_partition) = partitioner.partitionProfiles(profiles)
print("Users -> partitions:\n{}".format(user_to_partition))
try:
outdir = os.path.join(args.outdir, args.pname);
os.makedirs(outdir)
except os.error as e:
if e.errno != errno.EEXIST:
print("makedirs failed on: {}".format(outdir))
print("error: {}".format(e))
else:
print("Removing previous partition output directory: {}".format(outdir))
shutil.rmtree(outdir)
os.makedirs(outdir)
print("Writing partition information to {}".format(outdir))
# Write all partition profiles to disk
for pnum, profile in partitions.items():
fname = os.path.join(outdir, str(pnum) + ".json")
js.write_json(profile, filename=fname)
user_map = os.path.join(outdir, 'user_map.json')
js.write_json(user_to_partition, filename=user_map)
