def test_instantiate_matrix_set(self):
periods = 5
a = tm.TransitionMatrixSet(dimension=2, periods=periods)
self.assertEqual(a.temporal_type, 'Incremental')
self.assertAlmostEqual(a.entries[0][0, 0], 1.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a.entries[periods-1][0, 0], 1.0, places=ACCURATE_DIGITS, msg=None, delta=None)
pass
def test_set_cumulate_incremental(self):
a = tm.TransitionMatrix(values=[[0.6, 0.2, 0.2], [0.2, 0.6, 0.2], [0.2, 0.2, 0.6]])
a_set = tm.TransitionMatrixSet(values=a, periods=3, method='Copy', temporal_type='Incremental')
b_set = a_set
b_set.cumulate()
b_set.incremental()
self.assertAlmostEqual(a_set.entries[2][0, 0], b_set.entries[2][0, 0], places=ACCURATE_DIGITS, msg=None, delta=None)
pass
sorted_data = data.sort_values(['ID', 'Timestep'], ascending=[True, True])
print(myState.validate_dataset(dataset=sorted_data))
# compute confidence interval using goodman method at 95% confidence level
myEstimator = es.CohortEstimator(states=myState,
ci={
'method': 'goodman',
'alpha': 0.05
})
result = myEstimator.fit(sorted_data)
# Print confidence intervals
myEstimator.summary()
# Print the estimated results
myMatrixSet = tm.TransitionMatrixSet(values=result,
temporal_type='Incremental')
print(myMatrixSet.temporal_type)
myMatrixSet.print()
elif example == 2:
# Example: IFRS 9 Style Migration Matrix
# Load historical data into pandas frame
# Format:
# discrete time grid (already arranged in cohorts)
# Step 1
# Load the data set into a pandas frame
# Make sure state is read as a string and not as integer
# Fifth synthetic data example: IFRS 9 Migration Matrix
print(">>> Step 1")
data = pd.read_csv(dataset_path + 'synthetic_data5.csv',
""" Example of using transitionMatrix adjust to generate a full multi-period matrix set
Input data are processed Standard and Poor's matrices for a selection of cumulative observation points
"""
import transitionMatrix as tm
from scipy.linalg import expm
import numpy as np
from transitionMatrix import source_path
dataset_path = source_path + "datasets/"
print(
"-- Loading multi-period transitional matrices (cumulative mode) from json file"
)
SnP_Set0 = tm.TransitionMatrixSet(json_file=dataset_path +
"sp_NR_adjusted.json",
temporal_type='Cumulative')
print(SnP_Set0.validate())
# SnP_Set0.print(format='Percent')
print("-- Set the timesteps at which we have matrix observations")
# We skip the 15 and 20 year time points as they require further processing
SnP_Set0.timesteps = [1, 2, 3, 5, 7, 10]
print(SnP_Set0.timesteps)
# we will store the results here
timesteps = SnP_Set0.timesteps[len(SnP_Set0.timesteps) - 1]
SnP = tm.TransitionMatrixSet(dimension=8, periods=timesteps)
print("-- Fill in the gaps between periods")
t_list = SnP_Set0.timesteps
def test_matrix_set_load_csv(self):
a = tm.TransitionMatrixSet(csv_file=dataset_path + "sp_1981-2016.csv",
temporal_type='Cumulative')
a.validate()
self.assertEqual(a.periods, [1, 2, 3, 5, 7, 10, 15, 20])
def test_set_validation(self):
a = tm.TransitionMatrixSet(dimension=2, periods=5)
self.assertEqual(a.validate(), True)
# Initialize a single period transition matrix
# Example 1: Generic -> Typical Credit Rating Transition Matrix
# Example 2: Minimal -> Three state transition matrix
M = tm.TransitionMatrix(values=Generic)
# The size of the rating scale
Ratings = M.dimension
# The Default (absorbing state)
Default = Ratings - 1
# Lets extend the matrix into multi periods
Periods = 10
T = tm.TransitionMatrixSet(values=M,
periods=Periods,
method='Power',
temporal_type='Cumulative')
# Initialize a threshold set
As = ThresholdSet(TMSet=T)
for ri in range(0, Ratings):
print("RI: ", ri)
As.fit(AR_Model, ri)
lines = []
ri = 3
for rf in range(0, Ratings):
for k in range(0, Periods):
if rf != ri:
value = As.A[ri, rf, k]
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
""" Example of using transitionMatrix to detect and solve various pathologies that might be affecting transition
matrix data
"""
import transitionMatrix as tm
import numpy as np
from transitionMatrix import dataset_path
print("> Loading historical multi-period transitional matrices (cumulative mode) from csv file")
SnP_Set0 = tm.TransitionMatrixSet(csv_file=dataset_path + "sp_1981-2016.csv", temporal_type='Cumulative')
print("> Validate")
print(SnP_Set0.validate())
print(
"> We detect dimensionality problems. The matrices are not square (missing the trivial Default and NR transitions)")
print("> We must fix that to proceed. Augment matrices in set by fixing Default and NR transitions")
C_Vals = []
for matrix in SnP_Set0.entries:
C = tm.TransitionMatrix(values=np.resize(matrix, (9, 9)))
# set the migration from NR or D state to a rated state to zero
C[7, 0:9] = 0.0
C[8, 0:9] = 0.0
# set the probability of remaining to a D state to unity
C[7, 7] = 100.0
# set the probability of remaining to an NR state to unity
C[8, 8] = 100.0
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
""" Example of using transitionMatrix adjust the NR state (not-rated).
Input data are the Standard and Poor's historical data (1981 - 2016) for corporate credit rating migrations
"""
import transitionMatrix as tm
from transitionMatrix import source_path
dataset_path = source_path + "datasets/"
print(
"> Load multi-period transitional matrices (cumulative mode) from json file"
)
SnP_Set0 = tm.TransitionMatrixSet(json_file=dataset_path + "sp_1981-2016.json",
temporal_type='Cumulative')
print("> Valid Input Matrix? ", SnP_Set0.validate())
print("> Remove NR transitions and redistribute to other states")
SnP_Set1 = SnP_Set0.remove(8, "noninform")
print("> Valid Output Matrix? ", SnP_Set1.validate())
#
# Hurrah, we have an NR adjusted matrix set. Lets save it
#
SnP_Set1.to_json(dataset_path + 'sp_NR_adjusted.json', accuracy=5)
# limitations under the License.
""" Examples using transitionMatrix to perform operations with transition matrix sets sequences
"""
import transitionMatrix as tm
from transitionMatrix.creditratings.predefined import Generic as T1
print("-- Lets seed the set with a 3x3 matrix")
A = tm.TransitionMatrix(
values=[[0.6, 0.2, 0.2], [0.2, 0.6, 0.2], [0.2, 0.2, 0.6]])
print(A)
print("-- Identical future period transitions in incremental mode")
A_Set = tm.TransitionMatrixSet(values=A,
periods=3,
method='Copy',
temporal_type='Incremental')
print(A_Set.entries)
print(
"-- Identical future period transitions in cumulative mode using the power method"
)
B_Set = tm.TransitionMatrixSet(values=A,
periods=3,
method='Power',
temporal_type='Cumulative')
print(B_Set.entries)
print("-- Lets instantiate the set directly using a list of matrices")
C_Vals = [[[0.75, 0.25], [0.0, 1.0]], [[0.75, 0.25], [0.0, 1.0]]]
C_Set = tm.TransitionMatrixSet(values=C_Vals, temporal_type='Incremental')
definition = [('A', "Grade A"), ('B', "Grade B"), ('C', "Grade C"),
('D', "Grade D"), ('E', "Grade E"), ('F', "Grade F"),
('G', "Grade G"), ('H', "Delinquent"), ('I', "Charged Off"),
('J', "Repaid")]
myState = tm.StateSpace(definition)
# Load the data sets into a pandas frame in sequence
# Check matrix_lendingclub.py for comments
matrix_set = []
for letter in ['a', 'b', 'c', 'd']:
# store the derived one-period matrices in a list
data = pd.read_csv(dataset_path + 'LoanStats3' + letter + '_Step2.csv')
myEstimator = es.SimpleEstimator(states=myState,
ci={
'method': 'goodman',
'alpha': 0.05
})
result = myEstimator.fit(data)
myEstimator.summary()
myMatrix = tm.TransitionMatrix(result)
myMatrix[7, 9] = 1.0
myMatrix[8, 9] = 1.0
myMatrix[9, 9] = 1.0
matrix_set.append(myMatrix)
# collect all matrices in a matrix set
LC_Set = tm.TransitionMatrixSet(values=matrix_set, temporal_type='Incremental')
LC_Set.print_matrix()
if example == 1:
M = tm.TransitionMatrix(values=Generic)
# Lets take a look at the values
M.print()
M.validate()
# The size of the rating scale
Ratings = M.dimension
# The Default (absorbing state)
Default = Ratings - 1
# Lets extend the matrix into multi periods
Periods = 5
T = tm.TransitionMatrixSet(values=M,
periods=Periods,
method='Power',
temporal_type='Cumulative')
# Initialize a threshold set
As = ThresholdSet(TMSet=T)
print("> Fit Multiperiod Thresholds")
for ri in range(0, Ratings):
print("RI: ", ri)
As.fit(AR_Model, ri)
print(
"> Validate Multiperiod Thresholds against Input Transition Matrix Set"
)
Q = As.validate(AR_Model)
def __init__(self, ratings=None, periods=None, TMSet=None, json_file=None):
""" Create a new threshold set. Different options for initialization are:
* providing shape values (Ratings, Periods)
* providing a transition matrix set
* providing a file URL
:param ratings: size of the transition matrix
:param periods: number of periods (equally spaced)
:param TMSet: a TransitionMatrix Set object (optional)
:type ratings: int
:type periods: int
:type TMSet: object
:returns: returns a ThresholdSet object
:rtype: object
.. note:: The transition matrix set
:Example:
Instantiate a threshold set directly using an existing transition matrix set
A = tm.thresholds.ThresholdSet(TMSet=T)
"""
if (ratings and periods) is not None:
self.A = np.zeros((ratings, ratings, periods))
self.T = tm.TransitionMatrixSet(dimension=ratings,
periods=periods,
temporal_type='Cumulative')
elif TMSet is not None:
self.ratings = TMSet.entries[0].dimension
self.periods = len(TMSet.periods)
# Grid and accuracy settings
self.grid_size = settings.GRID_POINTS
self.precision = settings.PRECISION
self.scale = settings.SCALE
# Thresholds
self.A = np.zeros(shape=(self.ratings, self.ratings, self.periods),
dtype='double')
# Transition Matrix Set
self.T = TMSet
# Process Survival Density per period
self.f = np.zeros(shape=(self.grid_size, self.periods,
self.ratings),
dtype='double')
# Solution Grid
self.grid = np.zeros(shape=(self.grid_size, self.periods,
self.ratings),
dtype='double')
self.grid_step = np.zeros(shape=(self.periods, self.ratings),
dtype='double')
# Set the max grid value using the scale parameter (heuristic rule)
self.grid_max = np.zeros(shape=(self.periods, ), dtype='double')
elif json_file is not None:
self.from_json(json_file)
else:
raise ValueError
def validate(self, AR_Model):
""" Validate calculated Thresholds given autoregressive model
The comparison is accomplished by producing the implied transition matrix and setting against
the input set
.. Todo:: Automate the comparison when a new set is produced
"""
# Initialize a transition Matrix set container to hold the results
Q = tm.TransitionMatrixSet(dimension=self.ratings,
periods=self.periods,
temporal_type='Cumulative')
# The Default (absorbing state)
Default = self.ratings - 1
# AR Process parameters
mu = AR_Model['Mu']
phi_1 = AR_Model['Phi'][0]
x_0 = AR_Model['Initial Conditions'][0]
# Validate the transition rates for all initial ratings
for ri in range(0, Default):
# ========== PERIOD LOOP =========
for k in range(0, self.periods):
# rf = Default is separate case
# survival probability during k
integral = np.trapz(self.f[:, k, ri], self.grid[:, k, ri],
self.grid_step[k, ri])
Q.entries[k][ri, Default] = 1.0 - integral
# Testing Transitions to top (rf = 0) rating
# If starting from a rating below 0
if ri > 0:
p_grid = ma.masked_less(self.grid[:, k, ri], self.A[ri, 0,
k])
p_f = ma.masked_array(self.f[:, k, ri], p_grid.mask)
integral = np.trapz(p_f, p_grid, self.grid_step[k, ri])
Q.entries[k][ri, 0] = integral
# If starting at rating 0
else:
p_grid = ma.masked_less(self.grid[:, k, ri], self.A[0, 1,
k])
p_f = ma.masked_array(self.f[:, k, ri], p_grid.mask)
integral = np.trapz(p_f, p_grid, self.grid_step[k, ri])
Q.entries[k][0, 0] = integral
# Testing other transitions
for rf in range(1, self.ratings - 1):
# Staying in same rating
if rf == ri:
p_grid = ma.masked_outside(self.grid[:, k, ri],
self.A[ri, rf - 1, k],
self.A[ri, rf + 1, k])
# Upgrade
elif rf < ri:
p_grid = ma.masked_outside(self.grid[:, k, ri],
self.A[ri, rf, k],
self.A[ri, rf - 1, k])
# Downgrade
elif rf > ri:
p_grid = ma.masked_outside(self.grid[:, k, ri],
self.A[ri, rf, k],
self.A[ri, rf + 1, k])
p_f = ma.masked_array(self.f[:, k, ri], p_grid.mask)
integral = np.trapz(p_f, p_grid, self.grid_step[k, ri])
Q.entries[k][ri, rf] = integral
print(
'=============================================================================='
)
print(
' Transition Matrix Validation Results '
)
print(
'=============================================================================='
)
print('Number of Rating States: ', self.ratings)
print('Number of Periods: ', self.periods)
print(
'------------------------------------------------------------------------------'
)
print(
'T is the input matrix, Q is reconstructed matrix, E is the absolute error'
)
print(
'------------------------------------------------------------------------------'
)
for ri in range(0, self.ratings):
print('From Initial Rating State', ri)
print(
'------------------------------------------------------------------------------'
)
for rf in range(0, self.ratings):
print('T ', end=' ')
print('{0:3}'.format(rf), end='')
print(' | ', end='')
for k in range(0, self.periods):
val = self.T.entries[k][ri, rf]
print('{:06.4f}'.format(val), end=' ')
print('')
print('Q ', end=' ')
print('{0:3}'.format(rf), end='')
print(' | ', end='')
for k in range(0, self.periods):
val = Q.entries[k][ri, rf]
print('{:06.4f}'.format(val), end=' ')
print('')
print('E ', end=' ')
print('{0:3}'.format(rf), end='')
print(' | ', end='')
for k in range(0, self.periods):
val = self.T.entries[k][ri, rf] - Q.entries[k][ri, rf]
print('{:06.4f}'.format(val), end=' ')
print('')
print(
'..............................................................................'
)
print(
'=============================================================================='
)
return Q
