def test_fail(self): """Test fail method """ spinner = HaloNotebook() spinner.start('foo') spinner.fail() output = self._get_test_output(spinner)['text'] pattern = re.compile(r'(✖|×) foo', re.UNICODE) self.assertRegexpMatches(output[-1], pattern) spinner.stop()
def model_explanation(data_df, prediction_column, problem_type, snr='auto', file_name=None): """ .. _model-explanation: Analyzes the variables that a model relies on the most in a brute-force fashion. The first variable is the variable the model relies on the most. The second variable is the variable that complements the first variable the most in explaining model decisions etc. Running performances should be understood as the performance achievable when trying to guess model predictions using variables with selection order smaller or equal to that of the row. When :code:`problem_type=None`, the nature of the supervised learning problem (i.e. regression or classification) is inferred from whether or not :code:`prediction_column` is categorical. Parameters ---------- data_df : pandas.DataFrame The pandas DataFrame containing the data. prediction_column : str The name of the column containing true labels. problem_type : None | 'classification' | 'regression' The type of supervised learning problem. When None, it is inferred from the column type and the number of distinct values. file_name : None | str A unique identifier characterizing data_df in the form of a file name. Do not set this unless you know why. Returns ------- result : pandas.DataFrame The result is a pandas.Dataframe with columns (where applicable): * :code:`'Selection Order'`: The order in which the associated variable was selected, starting at 1 for the most important variable. * :code:`'Variable'`: The column name corresponding to the input variable. * :code:`'Running Achievable R-Squared'`: The highest :math:`R^2` that can be achieved by a classification model using all variables selected so far, including this one. * :code:`'Running Achievable Accuracy'`: The highest classification accuracy that can be achieved by a classification model using all variables selected so far, including this one. * :code:`'Running Achievable RMSE'`: The highest classification accuracy that can be achieved by a classification model using all variables selected so far, including this one. .. admonition:: Theoretical Foundation Section :ref:`a) Model Explanation`. """ assert prediction_column in data_df.columns, 'The label column should be a column of the dataframe.' assert problem_type.lower() in ['classification', 'regression'] if problem_type.lower() == 'regression': assert np.can_cast(data_df[prediction_column], float), 'The prediction column should be numeric' k = 0 kp = 0 max_k = 100 file_name = upload_data(data_df, file_name=file_name) spinner = Halo(text='Waiting for results from the backend.', spinner='dots') spinner.start() if file_name: job_id = EXPLANATION_JOB_IDS.get( (file_name, prediction_column, problem_type), None) if job_id: api_response = APIClient.route( path='/wk/variable-selection', method='POST', \ file_name=file_name, target_column=prediction_column, \ problem_type=problem_type, timestamp=int(time()), job_id=job_id, \ snr=snr) else: api_response = APIClient.route( path='/wk/variable-selection', method='POST', \ file_name=file_name, target_column=prediction_column, \ problem_type=problem_type, timestamp=int(time()), snr=snr) initial_time = time() while api_response.status_code == requests.codes.ok and k < max_k: if kp % 2 != 0: sleep(2 if kp < 5 else 10 if k < max_k - 4 else 300) kp += 1 k = kp // 2 else: try: response = api_response.json() if 'job_id' in response: job_id = response['job_id'] EXPLANATION_JOB_IDS[(file_name, prediction_column, problem_type)] = job_id sleep(2 if kp < 5 else 10 if k < max_k - 4 else 300) kp += 1 k = kp // 2 # Note: it is important to pass the job_id to avoid being charged twice for the work. api_response = APIClient.route( path='/wk/variable-selection', method='POST', \ file_name=file_name, target_column=prediction_column, \ problem_type=problem_type, timestamp=int(time()), job_id=job_id, \ snr=snr) try: response = api_response.json() if 'eta' in response: progress_text = '%s%% Completed.' % response[ 'progress_pct'] if 'progress_pct' in response else '' spinner.text = 'Waiting for results from the backend. ETA: %s. %s' % ( response['eta'], progress_text) except: pass if ('job_id' not in response) or ('selection_order' in response): duration = int(time() - initial_time) duration = str( duration) + 's' if duration < 60 else str( duration // 60) + 'min' result = {} if 'selection_order' in response: result['Selection Order'] = response[ 'selection_order'] if 'variable' in response: result['Variable'] = response['variable'] if 'r-squared' in response: result['Running Achievable R-Squared'] = response[ 'r-squared'] if 'log-likelihood' in response: result[ 'Running Achievable Log-Likelihood Per Sample'] = response[ 'log-likelihood'] if 'rmse' in response and problem_type.lower( ) == 'regression': result['Running Achievable RMSE'] = response[ 'rmse'] if 'accuracy' in response and problem_type.lower( ) == 'classification': result['Running Achievable Accuracy'] = response[ 'accuracy'] result = pd.DataFrame.from_dict(result) if 'selection_order' in response: result.set_index('Selection Order', inplace=True) spinner.text = 'Received results from the backend after %s.' % duration spinner.succeed() return result except: logging.exception( '\nModel explanation failed. Last HTTP code: %s, Content: %s' % (api_response.status_code, api_response.content)) spinner.text = 'The backend encountered an unexpected error we are looking into. Please try again later.' spinner.fail() return None if api_response.status_code != requests.codes.ok: spinner.text = 'The backend is taking longer than expected. Please try again later' spinner.fail() try: response = api_response.json() if 'message' in response: logging.error('\n%s' % response['message']) except: logging.error( '\nModel explanation failed. Last HTTP code: %s, Content: %s' % (api_response.status_code, api_response.content)) raise LongerThanExpectedException( 'The backend is taking longer than expected, but rest reassured your task is still running. Please try again later to retrieve your results.' ) return None
def data_valuation(data_df, target_column, problem_type, snr='auto', include_mutual_information=False, file_name=None): """ .. _data-valuation: Estimate the highest performance metrics achievable when predicting the :code:`target_column` using all other columns. When :code:`problem_type=None`, the nature of the supervised learning problem (i.e. regression or classification) is inferred from whether or not :code:`target_column` is categorical. Parameters ---------- data_df : pandas.DataFrame The pandas DataFrame containing the data. target_column : str The name of the column containing true labels. problem_type : None | 'classification' | 'regression' The type of supervised learning problem. When None, it is inferred from the column type and the number of distinct values. include_mutual_information : bool Whether to include the mutual information between target and explanatory variables in the result. file_name : None | str A unique identifier characterizing data_df in the form of a file name. Do not set this unless you know why. Returns ------- achievable_performance : pandas.Dataframe The result is a pandas.Dataframe with columns (where applicable): * :code:`'Achievable Accuracy'`: The highest classification accuracy that can be achieved by a model using provided inputs to predict the label. * :code:`'Achievable R-Squared'`: The highest :math:`R^2` that can be achieved by a model using provided inputs to predict the label. * :code:`'Achievable RMSE'`: The lowest Root Mean Square Error that can be achieved by a model using provided inputs to predict the label. * :code:`'Achievable Log-Likelihood Per Sample'`: The highest true log-likelihood per sample that can be achieved by a model using provided inputs to predict the label. .. admonition:: Theoretical Foundation Section :ref:`1 - Achievable Performance`. """ assert target_column in data_df.columns, 'The label column should be a column of the dataframe.' assert problem_type.lower() in ['classification', 'regression'] if problem_type.lower() == 'regression': assert np.can_cast(data_df[target_column], float), 'The target column should be numeric' k = 0 max_k = 100 file_name = upload_data(data_df, file_name=file_name) spinner = Halo(text='Waiting for results from the backend.', spinner='dots') spinner.start() if file_name: job_id = VALUATION_JOB_IDS.get( (file_name, target_column, problem_type, snr), None) if job_id: api_response = APIClient.route( path='/wk/data-valuation', method='POST', file_name=file_name, target_column=target_column, \ problem_type=problem_type, \ timestamp=int(time()), job_id=job_id, \ snr=snr) else: api_response = APIClient.route( path='/wk/data-valuation', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, timestamp=int(time()), \ snr=snr) initial_time = time() while api_response.status_code == requests.codes.ok and k < max_k: try: response = api_response.json() if 'eta' in response: progress_text = '%s%% Completed.' % response[ 'progress_pct'] if 'progress_pct' in response else '' spinner.text = 'Waiting for results from the backend. ETA: %s. %s' % ( response['eta'], progress_text) if ('job_id' in response) and ('r-squared' not in response): job_id = response['job_id'] VALUATION_JOB_IDS[(file_name, target_column, problem_type, snr)] = job_id k += 1 sleep(15.) # Note: it is important to pass the job_id to avoid being charged twice for the same work. api_response = APIClient.route( path='/wk/data-valuation', method='POST', file_name=file_name, target_column=target_column, \ problem_type=problem_type, \ timestamp=int(time()), job_id=job_id, \ snr=snr) try: response = api_response.json() if 'eta' in response: progress_text = '%s%% Completed.' % response[ 'progress_pct'] if 'progress_pct' in response else '' spinner.text = 'Waiting for results from the backend. ETA: %s. %s' % ( response['eta'], progress_text) except: pass if ('job_id' not in response) or ('r-squared' in response): duration = int(time() - initial_time) duration = str(duration) + 's' if duration < 60 else str( duration // 60) + 'min' result = {} if 'r-squared' in response: result['Achievable R-Squared'] = [ response['r-squared'] ] if 'log-likelihood' in response: result['Achievable Log-Likelihood Per Sample'] = [ response['log-likelihood'] ] if 'rmse' in response and problem_type.lower( ) == 'regression': result['Achievable RMSE'] = [response['rmse']] if 'accuracy' in response and problem_type.lower( ) == 'classification': result['Achievable Accuracy'] = [response['accuracy']] if include_mutual_information and 'mi' in response: result['Mutual Information'] = [response['mi']] result = pd.DataFrame.from_dict(result) spinner.text = 'Received results from the backend after %s.' % duration spinner.succeed() return result except: logging.exception( '\nData valuation failed. Last HTTP code: %s' % api_response.status_code) spinner.text = 'The backend encountered an unexpected error we are looking into. Please try again later.' spinner.fail() return None if api_response.status_code != requests.codes.ok: spinner.text = 'The backend is taking longer than expected. Try again later.' spinner.fail() try: response = api_response.json() if 'message' in response: logging.error('\n%s' % response['message']) except: logging.error('\nData valuation failed. Last HTTP code: %s' % api_response.status_code) raise LongerThanExpectedException( 'The backend is taking longer than expected, but rest reassured your task is still running. Please try again later to retrieve your results.' ) return None
def data_driven_improvability(data_df, target_column, new_variables, problem_type, snr='auto', file_name=None): """ .. data-driven-improvability: Estimate the potential performance boost that a set of new explanatory variables can bring about. Parameters ---------- data_df : pandas.DataFrame The pandas DataFrame containing the data. target_column : str The name of the column containing true labels. new_variables : list The names of the columns to use as new explanatory variables. problem_type : None | 'classification' | 'regression' The type of supervised learning problem. When None, it is inferred from whether or not :code:`target_column` is categorical. file_name : None | str A unique identifier characterizing data_df in the form of a file name. Do not set this unless you know why. Returns ------- result : pandas.Dataframe The result is a pandas.Dataframe with columns (where applicable): * :code:`'Accuracy Boost'`: The classification accuracy boost that the new explanatory variables can bring about. * :code:`'R-Squared Boost'`: The :math:`R^2` boost that the new explanatory variables can bring about. * :code:`'RMSE Reduction'`: The reduction in Root Mean Square Error that the new explanatory variables can bring about. * :code:`'Log-Likelihood Per Sample Boost'`: The boost in log-likelihood per sample that the new explanatory variables can bring about. .. admonition:: Theoretical Foundation Section :ref:`3 - Model Improvability`. """ assert target_column in data_df.columns, 'The label column should be a column of the dataframe.' assert problem_type.lower() in ['classification', 'regression'] assert len(new_variables) > 0, 'New variables should be provided' for col in new_variables: assert col in data_df.columns, '%s should be a column in the dataframe' % col if problem_type.lower() == 'regression': assert np.can_cast(data_df[target_column], float), 'The target column should be numeric' k = 0 kp = 0 max_k = 100 file_name = upload_data(data_df, file_name=file_name) spinner = Halo(text='Waiting for results from the backend.', spinner='dots') spinner.start() if file_name: job_id = DD_IMPROVABILITY_JOB_IDS.get((file_name, target_column, str(new_variables), problem_type, snr), None) if job_id: api_response = APIClient.route( path='/wk/data-driven-improvability', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, new_variables=json.dumps(new_variables), \ job_id=job_id, timestamp=int(time()), snr=snr) else: api_response = APIClient.route( path='/wk/data-driven-improvability', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, new_variables=json.dumps(new_variables), \ timestamp=int(time()), snr=snr) initial_time = time() while api_response.status_code == requests.codes.ok and k < max_k: if kp%2 != 0: sleep(2 if kp<5 else 10 if k < max_k-4 else 300) kp += 1 k = kp//2 else: try: response = api_response.json() if 'job_id' in response: job_id = response['job_id'] DD_IMPROVABILITY_JOB_IDS[(file_name, target_column, str(new_variables), problem_type, snr)] = job_id sleep(2 if kp<5 else 10 if k < max_k-4 else 300) kp += 1 k = kp//2 api_response = APIClient.route( path='/wk/data-driven-improvability', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, new_variables=json.dumps(new_variables), \ timestamp=int(time()), snr=snr) try: response = api_response.json() if 'eta' in response: progress_text = '%s%% Completed.' % response['progress_pct'] if 'progress_pct' in response else '' spinner.text = 'Waiting for results from the backend. ETA: %s. %s' % (response['eta'], progress_text) except: pass if ('job_id' not in response) or ('r-squared-boost' in response): duration = int(time()-initial_time) duration = str(duration) + 's' if duration < 60 else str(duration//60) + 'min' result = {} if 'r-squared-boost' in response: result['R-Squared Boost'] = [response['r-squared-boost']] if 'log-likelihood-boost' in response: result['Log-Likelihood Per Sample Boost'] = [response['log-likelihood-boost']] if 'rmse-reduction' in response and problem_type.lower() == 'regression': result['RMSE Reduction'] = [response['rmse-reduction']] if 'accuracy-boost' in response and problem_type.lower() == 'classification': result['Accuracy Boost'] = [response['accuracy-boost']] result = pd.DataFrame.from_dict(result) spinner.text = 'Received results from the backend after %s' % duration spinner.succeed() return result except: spinner.text = 'The backend encountered an unexpected error we are looking into. Please try again later.' spinner.fail() return None if api_response.status_code != requests.codes.ok: spinner.text = 'The backend is taking longer than expected. Try again later.' spinner.fail() try: response = api_response.json() if 'message' in response: logging.error('\n%s' % response['message']) except: logging.error('\nData-driven improvability failed. Last HTTP code: %s' % api_response.status_code) return None
def model_driven_improvability(data_df, target_column, prediction_column, problem_type, snr='auto', file_name=None): """ .. model-driven-improvability: Estimate the extent to which a trained supervised learner may be improved in a model-driven fashion (i.e. without resorting to additional explanatory variables). Parameters ---------- data_df : pandas.DataFrame The pandas DataFrame containing the data. target_column : str The name of the column containing true labels. prediction_column : str The name of the column containing model predictions. problem_type : None | 'classification' | 'regression' The type of supervised learning problem. When None, it is inferred from whether or not :code:`target_column` is categorical. file_name : None | str A unique identifier characterizing data_df in the form of a file name. Do not set this unless you know why. Returns ------- result : pandas.Dataframe The result is a pandas.Dataframe with columns (where applicable): * :code:`'Lost Accuracy'`: The amount of classification accuracy that was irreversibly lost when training the supervised learner. * :code:`'Lost R-Squared'`: The amount of :math:`R^2` that was irreversibly lost when training the supervised learner. * :code:`'Lost RMSE'`: The amount of Root Mean Square Error that was irreversibly lost when training the supervised learner. * :code:`'Lost Log-Likelihood Per Sample'`: The amount of true log-likelihood per sample that was irreversibly lost when training the supervised learner. * :code:`'Residual R-Squared'`: For regression problems, this is the highest :math:`R^2` that may be achieved when using explanatory variables to predict regression residuals. * :code:`'Residual RMSE'`: For regression problems, this is the lowest Root Mean Square Error that may be achieved when using explanatory variables to predict regression residuals. * :code:`'Residual Log-Likelihood Per Sample'`: For regression problems, this is the highest log-likelihood per sample that may be achieved when using explanatory variables to predict regression residuals. .. admonition:: Theoretical Foundation Section :ref:`3 - Model Improvability`. """ assert target_column in data_df.columns, 'The label column should be a column of the dataframe.' assert prediction_column in data_df.columns, 'The prediction column should be a column of the dataframe.' assert problem_type.lower() in ['classification', 'regression'] if problem_type.lower() == 'regression': assert np.can_cast(data_df[target_column], float), 'The target column should be numeric' assert np.can_cast(data_df[prediction_column], float), 'The prediction column should be numeric' k = 0 kp = 0 max_k = 100 file_name = upload_data(data_df, file_name=file_name) spinner = Halo(text='Waiting for results from the backend.', spinner='dots') spinner.start() if file_name: job_id = MD_IMPROVABILITY_JOB_IDS.get((file_name, target_column, prediction_column, problem_type, snr), None) if job_id: api_response = APIClient.route( path='/wk/model-driven-improvability', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, prediction_column=prediction_column, \ job_id=job_id, timestamp=int(time()), snr=snr) else: api_response = APIClient.route( path='/wk/model-driven-improvability', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, prediction_column=prediction_column, \ timestamp=int(time()), snr=snr) while api_response.status_code == requests.codes.ok and k <= max_k: if kp%2 != 0: sleep(2 if kp<5 else 10 if k < max_k-4 else 300) kp += 1 k = kp//2 else: try: response = api_response.json() if 'job_id' in response: job_id = response['job_id'] MD_IMPROVABILITY_JOB_IDS[(file_name, target_column, prediction_column, problem_type, snr)] = job_id sleep(2 if kp<5 else 10 if k < max_k-4 else 300) kp += 1 k = kp//2 api_response = APIClient.route( path='/wk/model-driven-improvability', method='POST', \ file_name=file_name, target_column=target_column, \ problem_type=problem_type, prediction_column=prediction_column, \ job_id=job_id, timestamp=int(time()), snr=snr) try: response = api_response.json() if 'eta' in response: progress_text = '%s%% Completed.' % response['progress_pct'] if 'progress_pct' in response else '' spinner.text = 'Waiting for results from the backend. ETA: %s. %s' % (response['eta'], progress_text) except: pass if ('job_id' not in response) or ('lost-r-squared' in response): result = {} if 'lost-r-squared' in response: result['Lost R-Squared'] = [response['lost-r-squared']] if 'lost-log-likelihood' in response: result['Lost Log-Likelihood Per Sample'] = [response['lost-log-likelihood']] if 'lost-rmse' in response and problem_type.lower() == 'regression': result['Lost RMSE'] = [response['lost-rmse']] if 'lost-accuracy' in response and problem_type.lower() == 'classification': result['Lost Accuracy'] = [response['lost-accuracy']] if problem_type.lower() == 'regression': if 'residual-r-squared' in response: result['Residual R-Squared'] = [response['residual-r-squared']] if 'residual-log-likelihood' in response: result['Residual Log-Likelihood Per Sample'] = [response['residual-log-likelihood']] if 'residual-rmse' in response: result['Residual RMSE'] = [response['residual-rmse']] result = pd.DataFrame.from_dict(result) spinner.text = 'Received results from the backend after %s' % duration spinner.succeed() return result except: spinner.text = 'The backend encountered an unexpected error we are looking into. Please try again later.' spinner.fail() return None if api_response.status_code != requests.codes.ok: spinner.text = 'The backend is taking longer than expected. Please try again later.' spinner.fail() try: response = api_response.json() if 'message' in response: logging.error('\n%s' % response['message']) except: logging.error('\nModel-driven improvability failed. Last HTTP code: %s' % api_response.status_code) raise LongerThanExpectedException('The backend is taking longer than expected, but rest reassured your task is still running. Please try again later to retrieve your results.') return None
def information_adjusted_correlation(data_df, market_column, asset_column): """ Estimate the information-adjusted correlation between an asset return :math:`r` and the market return :math:`r_m`: :math:`\\text{IA-Corr}\\left(r, r_m \\right) := \\text{sgn}\\left(\\text{Corr}\\left(r, r_m \\right) \\right) \\left[1 - e^{-2I(r, r_m)} \\right]`, where :math:`\\text{sgn}\\left(\\text{Corr}\\left(r, r_m \\right) \\right)` the sign of the Pearson correlation coefficient. Unlike Pearson's correlation coefficient, which is 0 if and only if asset return and market return are **decorrelated** (i.e. they exhibit no linear relation), information-adjusted correlation is 0 if and only if market and asset returns are **statistically independent** (i.e. the exhibit no relation, linear or nonlinear). Parameters ---------- data_df : pandas.DataFrame The pandas DataFrame containing the data. market_column : str The name of the column containing market returns. asset_column : str The name of the column containing asset returns. Returns ------- result : float The information-adjusted correlation. """ assert market_column in data_df.columns, 'The market column should be a column of the dataframe.' assert asset_column in data_df.columns, 'The asset column should be a column of the dataframe.' assert np.can_cast(data_df[market_column], float), 'The market return column should be numeric' assert np.can_cast(data_df[asset_column], float), 'The asset return column should be numeric' k = 0 kp = 0 max_k = 100 spinner = Halo(text='Waiting for results from the backend.', spinner='dots') spinner.start() df = data_df[[market_column, asset_column]] file_name = upload_data(df) if file_name: job_id = IACORR_JOB_IDS.get(file_name, None) if job_id: api_response = APIClient.route( path='/wk/ia-corr', method='POST', file_name=file_name, market_column=market_column, \ asset_column=asset_column, \ timestamp=int(time()), job_id=job_id) else: api_response = APIClient.route( path='/wk/ia-corr', method='POST', \ file_name=file_name, market_column=market_column, \ asset_column=asset_column, \ timestamp=int(time())) initial_time = time() while api_response.status_code == requests.codes.ok and k < max_k: if kp % 2 != 0: sleep(2 if kp < 5 else 5 if k < max_k - 4 else 300) kp += 4 k = kp // 2 else: try: response = api_response.json() if 'job_id' in response: job_id = response['job_id'] IACORR_JOB_IDS[file_name] = job_id sleep(2 if kp < 5 else 5 if k < max_k - 4 else 300) kp += 4 k = kp // 2 # Note: it is important to pass the job_id to avoid being charged twice for the same work. api_response = APIClient.route( path='/wk/ia-corr', method='POST', file_name=file_name, market_column=market_column, \ asset_column=asset_column, \ timestamp=int(time()), job_id=job_id) try: response = api_response.json() if 'eta' in response: progress_text = '%s%% Completed.' % response[ 'progress_pct'] if 'progress_pct' in response else '' spinner.text = 'Waiting for results from the backend. ETA: %s. %s' % ( response['eta'], progress_text) except: pass if 'job_id' not in response: duration = int(time() - initial_time) duration = str( duration) + 's' if duration < 60 else str( duration // 60) + 'min' spinner.text = 'Received results from the backend in %s' % duration spinner.succeed() if 'ia-corr' in response: return response['ia-corr'] else: return np.nan except: spinner.text = 'The backend encountered an unexpected error we are looking into. Please try again later.' spinner.fail() logging.exception( '\nInformation-adjusted correlation failed. Last HTTP code: %s' % api_response.status_code) return None if api_response.status_code != requests.codes.ok: spinner.text = 'The backend is taking longer than expected. Please try again later.' spinner.fail() try: response = api_response.json() if 'message' in response: logging.error('\n%s' % response['message']) except: logging.error( '\nInformation-adjusted correlation failed. Last HTTP code: %s' % api_response.status_code) return None