def test_load_training_data_use_existing(self): logger.info( "Running Test Case: {id}".format(id=self.id().split('.')[-1])) existing_entries = self.test_data actual = ml.load_training_data(training_data=None, existing_entries=existing_entries) self.assertEqual(existing_entries, actual)
def enhance_transactions(self): # load training data self.training_data = ml.load_training_data( self.training_data, filter_training_data_by_account=self. filter_training_data_by_account, existing_entries=self.existing_entries) # convert training data to a list of TxnPostingAccounts self.converted_training_data = [ ml.TxnPostingAccount(t, p, pRef.account) for t in self.training_data for pRef in t.postings for p in t.postings if p.account != pRef.account ] # train the machine learning model self._trained = False if not self.converted_training_data: logger.warning("Cannot train the machine learning model " "because the training data is empty.") elif len(self.converted_training_data) < 2: logger.warning( "Cannot train the machine learning model " "because the training data consists of less than two elements." ) else: transformers = [] transformer_weights = {} transformers.append( ('narration', Pipeline([ ('getNarration', ml.GetNarration()), ('vect', CountVectorizer(ngram_range=(1, 3))), ]))) transformer_weights['narration'] = 0.8 transformers.append( ('account', Pipeline([ ('getReferencePostingAccount', ml.GetReferencePostingAccount()), ('vect', CountVectorizer(ngram_range=(1, 3))), ]))) transformer_weights['account'] = 0.8 distinctPayees = set( map(lambda trx: trx.txn.payee, self.converted_training_data)) if len(distinctPayees) > 1: transformers.append( ('payee', Pipeline([ ('getPayee', ml.GetPayee()), ('vect', CountVectorizer(ngram_range=(1, 3))), ]))) transformer_weights['payee'] = 0.5 transformers.append(( 'dayOfMonth', Pipeline([ ('getDayOfMonth', ml.GetDayOfMonth()), ('caster', ml.ArrayCaster()), # need for issue with data shape ]))) transformer_weights['dayOfMonth'] = 0.1 self.pipeline = Pipeline([ ('union', FeatureUnion(transformer_list=transformers, transformer_weights=transformer_weights)), ('svc', SVC(kernel='linear')), ]) logger.debug("About to train the machine learning model...") self.pipeline.fit( self.converted_training_data, ml.GetPostingAccount().transform(self.converted_training_data)) logger.info("Finished training the machine learning model.") self._trained = True if not self._trained: logger.warning( "Cannot generate predictions or suggestions " "because there is no trained machine learning model.") return self.imported_transactions # predict missing second postings self.transactions = self.imported_transactions if self.predict_second_posting: logger.debug( "About to generate predictions for missing second postings...") predicted_accounts: List[str] predicted_accounts = self.pipeline.predict( self.imported_transactions) self.transactions = [ ml.add_posting_to_transaction(*t_a) for t_a in zip(self.transactions, predicted_accounts) ] logger.debug( "Finished adding predicted accounts to the transactions to be imported." ) # suggest accounts that are likely involved in the transaction if self.suggest_accounts: # get values from the SVC decision function logger.debug( "About to generate suggestions about related accounts...") decision_values = self.pipeline.decision_function( self.imported_transactions) # add a human-readable class label (i.e., account name) to each value, and sort by value: suggestions = [[ account for _, account in sorted(list( zip(distance_values, self.pipeline.classes_)), key=lambda x: x[0], reverse=True) ] for distance_values in decision_values] # add the suggested accounts to each transaction: self.transactions = [ ml.add_suggested_accounts_to_transaction(*t_s) for t_s in zip(self.transactions, suggestions) ] logger.debug( "Finished adding suggested accounts to the transactions to be imported." ) return self.transactions
def enhance_transactions(self): # load training data self.training_data = ml.load_training_data( self.training_data, filter_training_data_by_account=self. filter_training_data_by_account, existing_entries=self.existing_entries) # train the machine learning model self._trained = False if not self.training_data: logger.warning("Cannot train the machine learning model " "because the training data is empty.") elif len(self.training_data) < 2: logger.warning( "Cannot train the machine learning model " "because the training data consists of less than two elements." ) else: self.pipeline = Pipeline([ ( 'union', FeatureUnion( transformer_list=[ ('narration', Pipeline([ ('getNarration', ml.GetNarration()), ('vect', CountVectorizer(ngram_range=(1, 3))), ])), ( 'payee', Pipeline([ # any existing payee, if one exists ('getPayee', ml.GetPayee()), ('vect', CountVectorizer(ngram_range=(1, 3))), ])), ( 'dayOfMonth', Pipeline([ ('getDayOfMonth', ml.GetDayOfMonth()), ('caster', ml.ArrayCaster() ), # need for issue with data shape ])), ], transformer_weights={ 'narration': 0.8, 'payee': 0.5, 'dayOfMonth': 0.1 })), ('svc', SVC(kernel='linear')), ]) logger.debug("About to train the machine learning model...") self.pipeline.fit(self.training_data, ml.GetPayee().transform(self.training_data)) logger.info("Finished training the machine learning model.") self._trained = True if not self._trained: logger.warning( "Cannot generate predictions or suggestions " "because there is no trained machine learning model.") return self.imported_transactions # predict payees self.transactions = self.imported_transactions if self.predict_payees: logger.debug("About to generate predictions for payees...") predicted_payees: List[str] predicted_payees = self.pipeline.predict(self.transactions) self.transactions = [ ml.add_payee_to_transaction( *t_p, overwrite=self.overwrite_existing_payees) for t_p in zip(self.transactions, predicted_payees) ] logger.debug( "Finished adding predicted payees to the transactions to be imported." ) # predict payees self.transactions = self.imported_transactions if self.predict_payees: logger.debug("About to generate predictions for payees...") predicted_payees: List[str] predicted_payees = self.pipeline.predict( self.imported_transactions) self.transactions = [ ml.add_payee_to_transaction( *t_p, overwrite=self.overwrite_existing_payees) for t_p in zip(self.imported_transactions, predicted_payees) ] logger.debug( "Finished adding predicted payees to the transactions to be imported." ) # suggest likely payees if self.suggest_payees: # get values from the SVC decision function logger.debug( "About to generate suggestions about likely payees...") decision_values = self.pipeline.decision_function( self.imported_transactions) # add a human-readable class label (i.e., payee's name) to each value, and sort by value: suggested_payees = [[ payee for _, payee in sorted(list( zip(distance_values, self.pipeline.classes_)), key=lambda x: x[0], reverse=True) ] for distance_values in decision_values] # add the suggested payees to each transaction: self.transactions = [ ml.add_suggested_payees_to_transaction(*t_p) for t_p in zip(self.transactions, suggested_payees) ] logger.debug( "Finished adding suggested payees to the transactions to be imported." ) return self.transactions
def test_load_training_data(self): logger.info( "Running Test Case: {id}".format(id=self.id().split('.')[-1])) test_data = ml.load_training_data(training_data=os.path.join( os.path.dirname(__file__), 'sample_training.beancount')) self.assertEqual(1, len(list(test_data)))