def test_process_explode():
data = load_binary()
users, cycles, active_days, tracking = data['users'], data['cycles'], data['active_days'], data['tracking']
print("Going over all cycles ~20k:")
fc = process_explode(tracking, cycles)
print(len(fc), ' data points in tracking pivot')
assert(len(fc)>600000)
def run():
data = load_binary()
# Extract features
user_feat_matrix = process_level2(data) # X
del user_feat_matrix['X']['user_id']
X = user_feat_matrix['X'].values
X[np.isnan(X)] = 0
Y = user_feat_matrix['Y']
Y.fillna(0, inplace=True)
del user_feat_matrix['X_all']['user_id']
X_all = user_feat_matrix['X_all'].values
X_all[np.isnan(X_all)] = 0
cols = list(Y.columns.values)
symptoms = [
'happy', 'pms', 'sad', 'sensitive_emotion', 'energized', 'exhausted',
'high_energy', 'low_energy', 'cramps', 'headache', 'ovulation_pain',
'tender_breasts', 'acne_skin', 'good_skin', 'oily_skin', 'dry_skin'
]
with open("result.txt", 'w') as f:
f.write("user_id,day_in_cycle,symptom,probability\n")
for symptom in symptoms:
print(symptom)
pipeline = Pipeline([
('remove_low_variance_features', VarianceThreshold(threshold=0.0)),
#('standard_scale', StandardScaler()),
('estimator', Lasso()),
])
param_grid = {'estimator__alpha': [.1, .3, .5, .7, .8]}
model = GridSearchCV(pipeline,
param_grid=param_grid,
n_jobs=4,
verbose=2)
model.fit(X, s_Y.values)
print("dumping...")
data_dir = 'data'
cycles0 = pd.read_csv(join(data_dir, 'cycles0.csv'))
c_length = {
k: v
for k, v in zip(cycles0.user_id.values,
cycles0.expected_cycle_length)
}
dump(symptom, model, X_all, c_length, data['users'].user_id)
def check_probability_access(data):
'''find probability_access'''
df_active = data['active_days']
df_cycles = data['cycles']
access_prob = []
for i in range(1, 30):
access_prob.append((df_active['day_in_cycle'] == i).sum() / (
df_cycles['cycle_length'][df_cycles['cycle_length'] >= i]).count())
# access_prob.plot(X)
return access_prob
df = pd.read_csv('result.txt')
# now is done until 15 day, afterwords our predictions are wrong
daily_profiles = create_profile_for_symptoms(df, date_range=15)
data = load_binary()
access_profile = check_probability_access(data)
plt.plot(access_profile[0:29]) # probability of access
for symptom in symptoms:
real_prob = daily_profiles[symptom].copy()
for i in range(15):
real_prob.loc[i] = real_prob.loc[i] / access_profile[i]
plt.plot(real_prob)
def test_load_binary():
data = load_binary()
users, cycles, active_days, tracking = data['users'], data['cycles'], data[
'active_days'], data['tracking']
print(len(users), 'users loaded')
assert (len(users) > 3000)