def data_ingestion(): # Create/Update database for one hour time window with 1 minute intervals. for idx in xrange(1, 61): # Download data req = download_bike_data() # Normalize Station List data df_stationlist = normalized_stationbean_data(req) # json data json_stationlist = get_stationlist_info_json(req) # Get execution time exec_time = get_execution_time(req) ''' Run inserts and updates ''' # Create tables and bulk insert into reference table once. if idx == 1: output_data_keys(req) # For information purposes alone. create_tables(df_stationlist['id']) bulk_insert_citibike_ref_table() insert_update_available_bike_table(exec_time, json_stationlist) # Simple plots to observe data print "PLOT AND SAVE IMAGES FOR RUN: {}".format(idx) plot_and_save_hist(df_stationlist, 'availableBikes', 'figures/citibike', 'available_bikes', 'png', idx) plot_and_save_hist(df_stationlist, 'totalDocks', 'figures/citibike', 'total_docks', 'png', idx) plot_and_save_hist(df_stationlist, 'testStation', 'figures/citibike', 'test_station', 'png', idx) # Spinning cursor to wait for 60 seconds. spinning_cursor.spinning_cursor(60)
# GDP Men Total Women # GDP 1.000000 0.495794 0.479050 0.497923 # Men 0.495794 1.000000 0.971663 0.942572 # Total 0.479050 0.971663 1.000000 0.977217 # Women 0.497923 0.942572 0.977217 1.000000 # print """ FINAL ANALYSIS: We observe a weak correlation between education attainment and GDP. The correlation coefficients are closer to 0 than 1. This shows there is a greater scatter of the data points from the fitted line. At this point we cannot not conclude any direct relationship. [Data Analysis and Correlation of Education to GDP data] ==> End """ ############# RUN MAIN ########################## if __name__ == '__main__': df_education_result, min_year, max_year = data_ingestion_education_info() sc.spinning_cursor(3) dataingestion_worldbankgdp_info(min_year, max_year) sc.spinning_cursor(3) df_gdp_result = build_dataframe_gdp() sc.spinning_cursor(3) data_analysis_and_correlation(df_education_result, df_gdp_result) sc.spinning_cursor(3)