def load_models_csv(self,filepath, model_no = None): """ Load predictions from an individual sub model into a dataframe stored in the sub_models list, if no model_no is given then load data into next available index. Valid source is a CSV file. """ try: if model_no == None: model_no = len(self.sub_models) self.sub_models.append(data_io.load_flatfile_to_df(filepath, delimiter='')) else: self.sub_models[model_no]=data_io.load_flatfile_to_df(filepath, delimiter='') utils.info('Model loaded into index %s' % str(model_no)) except IndexError: raise Exception('Model number does not exist. Model number given, %s, is out of index range.' % str(model_no))
def load_models_csv(self, filepath, model_no=None): """ Load predictions from an individual sub model into a dataframe stored in the sub_models list, if no model_no is given then load data into next available index. Valid source is a CSV file. """ try: if model_no == None: model_no = len(self.sub_models) self.sub_models.append( data_io.load_flatfile_to_df(filepath, delimiter='')) else: self.sub_models[model_no] = data_io.load_flatfile_to_df( filepath, delimiter='') utils.info('Model loaded into index %s' % str(model_no)) except IndexError: raise Exception( 'Model number does not exist. Model number given, %s, is out of index range.' % str(model_no))
def main(): log.info('********New program instance started********') #-------------Load Environment----------------------# #Get program settings and model settings from SETTINGS.json file in root directory settings, model_settings = utils.load_settings() #If not using cached data, then load raw data, clean/munge it, create hand-crafted features, slice it for CV if settings['use_cached_data'] == 'y': log.info('==========LOADING CACHED FEATURES===========') dfTrn = data_io.load_cached_object('dfTrn') dfTest = data_io.load_cached_object('dfTest') dfCV = data_io.load_flatfile_to_df('Data/CV.csv') else: #-------Data Loading/Cleaning/Munging------------# #Load the data log.info('===============LOADING DATA=================') dfTrn = data_io.load_flatfile_to_df(settings['file_data_train']) dfTest = data_io.load_flatfile_to_df(settings['file_data_test']) dfCV = data_io.load_flatfile_to_df('Data/CV.csv') #Clean/Munge the data log.info('=======CLEANING AND MUNGING DATA============') dfTrn = munge.clean(dfTrn) dfTest = munge.clean(dfTest) #-------Feature creation-------------------------# #Add all currently used hand crafted features to dataframes log.info('====CREATING HAND-CRAFTED DATA FEATURES=====') features.add(dfTrn) features.add(dfTest) #---------Data slicing/parsing--------------------------# #Split data for CV if settings['generate_cv_score'] == 'y': log.info('=====SPLITTING DATA FOR CROSS-VALIDATION====') if settings['cv_method'] == 'april': dfTrnCV, dfTestCV = munge.temporal_split(dfTrn, (2013, 04, 1)) elif settings['cv_method'] == 'march': #take an addtional week from February b/c of lack of remote_api source issues in March dfTrnCV, dfTestCV = munge.temporal_split(dfTrn, (2013, 02, 21)) elif settings['cv_method'] == 'list_split': #load stored list of data points and use those for CV dfCVlist = pd.DataFrame({'id': data_io.load_cached_object('Cache/cv_issue_ids.pkl'), 'dummy': 0}) dfTrnCV, dfTestCV = munge.list_split(dfTrn, dfCVlist) #--------------Modeling-------------------------# #If cached models exist then load them for reuse into segment_models. Then run through model_settings and for # each model where 'use_cached_model' is false then clear the cached model and recreate it fresh log.info('=========LOADING CACHED MODELS==============') segment_models = data_io.load_cached_object('segment_models') if segment_models == None: log.info('=========CACHED MODELS NOT LOADED===========') for model in model_settings: model['use_cached_model'] = 'n' segment_models = [] #Initialize new model for models not set to use cache log.info('=======INITIALIZING UN-CACHED MODELS========') index = 0 for model in model_settings: if model_settings[model]['use_cached_model'] == 'n': new_model = ensembles.Model(model_name=model,target=model_settings[model]['target'], segment=model_settings[model]['segment'], estimator_class=model_settings[model]['estimator_class'], estimator_params=model_settings[model]['estimator_params'], features=model_settings[model]['features'], postprocess_scalar=model_settings[model]['postprocess_scalar']) #Flag the model as not cached, so that it does not get skipped when running the modeling process new_model.use_cached_model='n' #Project specific model attributes not part of base class new_model.KNN_neighborhood_threshold=model_settings[model]['KNN_neighborhood_threshold'] new_model.sub_zip_neighborhood=model_settings[model]['sub_zip_neighborhood'] segment_models[index] = new_model log.info('Model %s intialized at index %i' % (model,index)) index += 1 #Cross validate all segment models (optional) if settings['export_cv_predictions_all_models'] == 'y' or settings['export_cv_predictions_new_models'] == 'y': log.info('============CROSS VALIDATION================') for model in segment_models[:]: #If model has cached CV predictions then skip predicting and just export them (if selected in settings) if hasattr(model,'dfCVPredictions'): log.info('Cached CV predictions found. Using cached CV predictions.') if settings['export_cv_predictions_all_models'] == 'y': data_io.save_predictions(model.dfCVPredictions,model.target,model_name=model.model_name, directory=settings['dir_submissions'], estimator_class=model.estimator_class, note='CV_list') else: print_model_header(model) #Prepare segment model: segment and create feature vectors for the CV data set dfTrn_Segment, dfTest_Segment = prepare_segment_model(dfTrnCV,dfTestCV,model) #Generate CV predictions train.cross_validate(model, settings, dfTrn_Segment, dfTest_Segment) #Cache the CV predictions as a dataframe stored in each segment model model.dfCVPredictions = dfTest_Segment.ix[:,['id',model.target]] if settings['export_cv_predictions_new_models'] == 'y': data_io.save_predictions(model.dfCVPredictions,model.target,model_name=model.model_name, directory=settings['dir_submissions'], estimator_class=model.estimator_class, note='CV_list') #Generate predictions on test set for all segment models (optional) if settings['export_predictions_all_models'] == 'y' or settings['export_predictions_new_models'] == 'y'\ or settings['export_predictions_total'] == 'y': log.info('=======GENERATING TEST PREDICTIONS==========') for model in segment_models[:]: #If model has cached test predictions then skip predicting and just export them (if selected in settings) if hasattr(model,'dfPredictions'): log.info('Cached test predictions found for model %s. Using cached predictions.' % model.model_name) if settings['export_predictions_all_models'] == 'y': data_io.save_predictions(model.dfPredictions,model.target,model_name=model.model_name, directory=settings['dir_submissions'], estimator_class=model.estimator_class,note='TESTset') else: print_model_header(model) #Prepare segment model: segment and create feature vectors for the full TEST data set dfTrn_Segment, dfTest_Segment = prepare_segment_model(dfTrn,dfTest,model) #Generate TEST set predictions model.predict(dfTrn_Segment, dfTest_Segment) if settings['export_predictions_all_models'] == 'y' or settings['export_predictions_new_models'] == 'y': data_io.save_predictions(model.dfPredictions,model.target,model_name=model.model_name, directory=settings['dir_submissions'], estimator_class=model.estimator_class,note='TESTset') log.info(utils.line_break()) #Cache the trained models and predictions to file (optional) if settings['export_cached_models'] == 'y': log.info('==========EXPORTING CACHED MODELS===========') data_io.save_cached_object(segment_models,'segment_models') #Merge each segment model's CV predictions into a master dataframe and export it (optional)----# if settings['export_cv_predictions_total'] == 'y': log.info('====MERGING CV PREDICTIONS FROM SEGMENTS====') dfTestPredictionsTotal = merge_segment_predictions(segment_models, dfTestCV, cv=True) #---Apply post process rules to master dataframe---# #Set all votes and comments for remote_api segment to 1 and 0 dfTestPredictionsTotal = dfTestPredictionsTotal.merge(dfTest.ix[:][['source','id']], on='id', how='left') for x in dfTestPredictionsTotal.index: if dfTestPredictionsTotal.source[x] == 'remote_api_created': dfTestPredictionsTotal.num_votes[x] = 1 dfTestPredictionsTotal.num_comments[x] = 0 #Export timestamp = datetime.now().strftime('%m-%d-%y_%H%M') filename = 'Submits/'+timestamp+'--bryan_CV_predictions.csv' dfTestPredictionsTotal.to_csv(filename) #Merge each segment model's TEST predictions into a master dataframe and export it (optional)----# if settings['export_predictions_total'] == 'y': log.info('===MERGING TEST PREDICTIONS FROM SEGMENTS===') dfTestPredictionsTotal = merge_segment_predictions(segment_models, dfTest) #---Apply post process rules to master dataframe---# #Set all votes and comments for remote_api segment to 1 and 0 dfTestPredictionsTotal = dfTestPredictionsTotal.merge(dfTest.ix[:][['source','id']], on='id', how='left') for x in dfTestPredictionsTotal.index: if dfTestPredictionsTotal.source[x] == 'remote_api_created': dfTestPredictionsTotal.num_votes[x] = 1 dfTestPredictionsTotal.num_comments[x] = 0 del dfTestPredictionsTotal['source'] #Export filename = 'bryan_test_predictions.csv' data_io.save_combined_predictions(dfTestPredictionsTotal, settings['dir_submissions'], filename) #End main log.info('********Program ran successfully. Exiting********')