# ### Export the Class Encoder Too
# In[51]:
joblib.dump(le, 'final_models/classes_encoder.pkl')
# **Export the X Data Columns to a List**
# We want to make sure we preserve the order so we can make sure the columns are in the correct order when using these models in production environment with new data.
# In[52]:
with open('final_models/modA_column_order.pkl', 'wb') as f:
pickle.dump(list(X.columns), f)
# --------------
# **Convert this notebook to a python file**
# In[6]:
sys.path.append(os.path.abspath('/Users/adriannaesh/Documents/ESH-Code/ficher/General_Resources/common_functions/'))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook('ESH_Modeling_A_Primary_Modeling.ipynb', 'ESH_Modeling_A_Primary_Modeling.py', main)
# # End
axis=1)
# **Note**: In python `0` and `False` are equal and `1` and `True` are equal - So some differences in those columns are acceptable.
# **Example (Adrianna Delete if you want)**
# In[68]:
print 0 == False
print 1 == True
# ----
# **Convert this notebook to a python file**
# In[2]:
sys.path.append(
os.path.abspath(
'/Users/adriannaesh/Documents/ESH-Code/ficher/General_Resources/common_functions/'
))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook('ESH_Featurizer_B_Future_Data_dk.ipynb',
'ESH_Featurizer_B_Future_Data_dk.py',
main)
# # End
# -------
df
# -------
# # Export Change Data
# In[44]:
raw_clean_mg.to_csv('model_data_versions/changecount_June16_17.csv',
encoding='utf-8')
# -------
# **Convert this notebook to a python file**
# In[2]:
sys.path.append(
os.path.abspath(
'/Users/adriannaesh/Documents/ESH-Code/ficher/General_Resources/common_functions/'
))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook('ESH_EDA_B_Raw_Clean_Compare_dk.ipynb',
'ESH_EDA_B_Raw_Clean_Compare_dk.py', main)
# # END
# --------
# # Example Use of the Saved Model
# In[22]:
joblib.load('final_models/col_diff_models/diff_function_model.pkl').score(
vld_X, validation_set.diff_function)
# ---------
# **Convert this notebook to a python file**
# In[2]:
sys.path.append(
os.path.abspath(
'/Users/adriannaesh/Documents/ESH-Code/ficher/General_Resources/common_functions/'
))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook(
'ESH_Modeling_B_Binary_Column_Difference_Prediction_dk.ipynb',
'ESH_Modeling_B_Binary_Column_Difference_Prediction_dk.py', main)
# # End
# --------
#
# ---------
#
# ---------
#composite_md = composite_md[modC_composite_col_order]
# ## Predict
# In[39]:
#modC_composite_model = joblib.load('final_models/composite_multiclass.pkl')
#modC_composite_model
# In[40]:
#pd.DataFrame(modC_composite_model.predict_proba(composite_md),columns=class_encoder.classes_)
# -----
# **Convert this notebook to a python file**
# In[4]:
sys.path.append(os.path.abspath('/Users/adriannaesh/Documents/ESH-Code/ficher/General_Resources/common_functions/'))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook('ESH_Prediction.ipynb', 'ESH_Prediction.py', main)
# # End
# In[58]:
print 'Before we Drop NA', final_model_data.shape
final_model_data = final_model_data.dropna()
print 'After we Drop NA', final_model_data.shape
# In[59]:
final_model_data.to_csv('data/model_data_output_June16_2017.csv', index=False)
# --------------
# **Convert this notebook to a python file**
# In[3]:
sys.path.append(
os.path.abspath(
'/Users/adriannaesh/Documents/ESH-Code/ficher/General_Resources/common_functions/'
))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook(
'ESH_Featurizer_A_Main_2016_Data_dk.ipynb',
'ESH_Featurizer_A_Main_2016_Data_dk.py', main)
# # End
# ----------
plt.annotate("$" + str(mu_true),
xy=(.9, mu_true + .9),
xytext=(.9, mu_true + .9),
color="grey")
plt.annotate("$" + str(mu_false),
xy=(1.9, mu_false + .9),
xytext=(1.9, mu_false + .9),
color="grey")
plt.annotate(str(mu_multiple) + "x",
xy=(1.5, (mu_true - mu_false) / 2 + mu_false),
xytext=(1.5, (mu_true - mu_false) / 2 + mu_false),
color='red',
size=20)
seaborn.despine(left=True, right=True)
seaborn.set_style("whitegrid", {'axes.grid': False})
plt.savefig("ia_cost_by_frn_bids.png")
plt.show()
##Model diagnostics
import statsmodels_ols_diagnostics as smd
get_ipython().magic(u'matplotlib inline')
smd.ols_model_diagnostics(est_ia_cost)
#will run only if using ipython notebook
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook('regression_ia_cost.ipynb',
'regression_ia_cost_qa.py', main)
# In[ ]:
districts_for_sc_reg.head()
districts_for_sc_reg.round(2)
if not os.path.exists(os.path.dirname('../../../data/interim/reg/')):
try:
os.makedirs(os.path.dirname('../../../data/interim/reg/'))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
districts_for_sc_reg.to_csv(
'../../../data/interim/reg/districts_for_sc_reg.csv')
districts_for_sc_reg_clean = districts_for_sc_reg.loc[
districts_for_sc_reg['exclude_from_ia_analysis'] == False]
districts_for_sc_reg_clean = districts_for_sc_reg_clean.loc[
districts_for_sc_reg_clean['exclude_from_ia_cost_analysis'] == False]
districts_for_sc_reg_clean.to_csv(
'../../../data/interim/reg/districts_for_sc_reg_clean.csv')
#will run only if using ipython notebook
sys.path.append(
os.path.abspath(
'/Users/sierra/Documents/ESH/ficher/General_Resources/common_functions'
))
import __main__ as main
import ipynb_convert
ipynb_convert.executeConvertNotebook('import_districts.ipynb',
'import_districts_qa.py', main)