def users_with_data_in_experiment_periods(userloads, user_ids):
"""Return a list of the users with no missing data in two specific,
hard-coded periods."""
return users_with_data_in_periods(userloads, user_ids,
ul.experiment_periods())
def users_with_data_in_experiment_periods(userloads, user_ids):
"""Return a list of the users with no missing data in two specific,
hard-coded periods."""
return users_with_data_in_periods(userloads, user_ids,
ul.experiment_periods())
sys.stdout.flush()
tempfeeder_nodup.read_all()
print "Done reading."
tempfeeder_dup.close()
tempfeeder_nodup.close()
print "Checking for highres meters..."
sys.stdout.flush()
highres = find_highres_meters(tempfeeder_nodup)
print "Narrowing to users with data in experiment periods..."
sys.stdout.flush()
highres_few_missing = \
users_with_data_in_experiment_periods(tempfeeder_nodup, highres)
print "Checking for duplicates."
sys.stdout.flush()
keepers = check_remove_userload_dupes(tempfeeder_nodup,
highres_few_missing)
print "Removing manually screened IDs"
keepers = remove_manually_screened_ids(keepers)
print len(keepers), "candidates for experiment:", keepers
path = "keepers.pickle"
print "Pickling the keepers to %s." % path
with open(path, "w") as f:
pickle.dump(keepers, f)
path = os.path.join(ul.DATA_DIR, "Experiment timeseries.pdf")
print "Plotting all selected timeseries to %s..." % path
plot_to_pdf(tempfeeder_nodup, keepers, path, ul.experiment_periods())
print "Storing experiment user IDs to HDF5 file."
for path in (tempfeeder_dup.path, tempfeeder_nodup.path):
store_user_ids(keepers, path, "cln_pred_exp_ids",
"IDs of meters selected for clean+predict experiment early 2012.")
tempfeeder_nodup.read_all()
print "Done reading."
tempfeeder_dup.close()
tempfeeder_nodup.close()
print "Checking for highres meters..."
sys.stdout.flush()
highres = find_highres_meters(tempfeeder_nodup)
print "Narrowing to users with data in experiment periods..."
sys.stdout.flush()
highres_few_missing = \
users_with_data_in_experiment_periods(tempfeeder_nodup, highres)
print "Checking for duplicates."
sys.stdout.flush()
keepers = check_remove_userload_dupes(tempfeeder_nodup,
highres_few_missing)
print "Removing manually screened IDs"
keepers = remove_manually_screened_ids(keepers)
print len(keepers), "candidates for experiment:", keepers
path = "keepers.pickle"
print "Pickling the keepers to %s." % path
with open(path, "w") as f:
pickle.dump(keepers, f)
path = os.path.join(ul.DATA_DIR, "Experiment timeseries.pdf")
print "Plotting all selected timeseries to %s..." % path
plot_to_pdf(tempfeeder_nodup, keepers, path, ul.experiment_periods())
print "Storing experiment user IDs to HDF5 file."
for path in (tempfeeder_dup.path, tempfeeder_nodup.path):
store_user_ids(
keepers, path, "cln_pred_exp_ids",
"IDs of meters selected for clean+predict experiment early 2012.")