def test_json_io(self):
a = tm.TransitionMatrix()
a.to_json("test.json")
b = tm.TransitionMatrix(json_file="test.json")
self.assertAlmostEqual(a[0, 0], b[0, 0], places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a[0, 1], b[0, 1], places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a[1, 0], b[1, 0], places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a[1, 1], b[1, 1], places=ACCURATE_DIGITS, msg=None, delta=None)
def test_validation(self):
a = tm.TransitionMatrix()
self.assertEqual(a.validate(), True)
b = tm.TransitionMatrix(values=[1.0, 3.0])
self.assertEqual(b.validate()[0][0], 'Matrix Dimensions Differ: ')
c = tm.TransitionMatrix(values=[[0.75, 0.25], [0.0, 0.9]])
self.assertEqual(c.validate()[0][0], 'Rowsum not equal to one: ')
d = tm.TransitionMatrix(values=[[0.75, 0.25], [-0.1, 1.1]])
self.assertEqual(d.validate()[0][0], 'Negative Probabilities: ')
def test_instantiate_matrix(self):
a = tm.TransitionMatrix()
self.assertAlmostEqual(a[0, 0], 1.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a[0, 1], 0.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a[1, 0], 0.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(a[1, 1], 1.0, places=ACCURATE_DIGITS, msg=None, delta=None)
b = tm.TransitionMatrix([[1.0, 3.0], [1.0, 4.0]])
self.assertAlmostEqual(b[0, 0], 1.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(b[0, 1], 3.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(b[1, 0], 1.0, places=ACCURATE_DIGITS, msg=None, delta=None)
self.assertAlmostEqual(b[1, 1], 4.0, places=ACCURATE_DIGITS, msg=None, delta=None)
def remove(self, state, method):
""" Remove a transition matrix state and distribute its probability mass to other states according to a prescribed method
:param state: the state to remove
:param method: the method to use
:type state: int
:type method: str
:returns: a transition matrix
.. todo:: Implement additional methods, for example a conservative approach where each NR is actually a default
"""
new_matrix = tm.TransitionMatrix(dimension=self.shape[0] - 1)
states = list(range(self.shape[0]))
del states[state]
# process all rows of the matrix except the state we remove
for i in states:
# probability mass to distribute
xp = self[i, state]
if 0.0 < xp < 1.0:
# process all columns of the matrix except the state we remove
w = xp / (1.0 - xp)
for j in states:
# weight of state among remaining states
new_matrix[i, j] = self[i, j] * (1.0 + w)
return new_matrix
def test_set_cumulate_incremental(self):
a = tm.TransitionMatrix(values=[[0.6, 0.2, 0.2], [0.2, 0.6, 0.2], [0.2, 0.2, 0.6]])
a_set = tm.TransitionMatrixSet(values=a, periods=3, method='Copy', temporal_type='Incremental')
b_set = a_set
b_set.cumulate()
b_set.incremental()
self.assertAlmostEqual(a_set.entries[2][0, 0], b_set.entries[2][0, 0], places=ACCURATE_DIGITS, msg=None, delta=None)
pass
def matrix_exponent(generator, t=1.0):
""" Compute the exponent of a transition matrix generator
:param t: the timescale parameter
:type t: float
:Example:
A = G.exponent()
"""
exponent = tm.TransitionMatrix(expm(t * generator))
return exponent
def power(self, n=1):
""" Raise a transition matrix to a desired power
:param n: the desired power
:type n: int
:Example:
B = A.power(10)
"""
a = self
for i in range(n - 1):
a = a * self
result = tm.TransitionMatrix(a)
return result
def test_roundtrip_identity(self):
definition = [('0', "A"), ('1', "B"), ('2', "C"), ('3', "D")]
myState = tm.StateSpace(definition)
input_data = dataset_generators.long_format(myState,
Identity,
n=100,
timesteps=2,
mode='Canonical')
compact_data = to_compact(input_data)
cohort_data, cohort_bounds = tm.utils.bin_timestamps(compact_data,
cohorts=1)
sorted_data = cohort_data.sort_values(['ID', 'Time'],
ascending=[True, True])
myEstimator = es.CohortEstimator(states=myState,
cohort_bounds=cohort_bounds,
ci={
'method': 'goodman',
'alpha': 0.05
})
result = myEstimator.fit(sorted_data,
labels={
'Time': 'Time',
'State': 'State',
'ID': 'ID'
})
myMatrix = tm.TransitionMatrix(myEstimator.average_matrix)
self.assertAlmostEqual(myMatrix[0, 0],
1.0,
places=ACCURATE_DIGITS,
msg=None,
delta=None)
self.assertAlmostEqual(myMatrix[1, 1],
1.0,
places=ACCURATE_DIGITS,
msg=None,
delta=None)
self.assertAlmostEqual(myMatrix[2, 2],
1.0,
places=ACCURATE_DIGITS,
msg=None,
delta=None)
self.assertAlmostEqual(myMatrix[2, 2],
1.0,
places=ACCURATE_DIGITS,
msg=None,
delta=None)
def cumulate(self):
""" Cumulate a transition matrix set from an incremental set
"""
if self.temporal_type is 'Cumulative':
print("Transition Matrix Set is already cumulated")
return
else:
val_set = []
periods = len(self.entries)
a = self.entries[0]
val_set.append(a)
an = a
for k in range(periods - 1):
an = an * a
an = tm.TransitionMatrix(an)
val_set.append(an)
self.entries = val_set
self.temporal_type = 'Cumulative'
return
def test_generator(self):
a = tm.TransitionMatrix([[1.0, 3.0], [1.0, 4.0]])
self.assertAlmostEqual(a[0, 0],
expm(a.generator())[0, 0],
places=ACCURATE_DIGITS,
msg=None,
delta=None)
self.assertAlmostEqual(a[0, 1],
expm(a.generator())[0, 1],
places=ACCURATE_DIGITS,
msg=None,
delta=None)
self.assertAlmostEqual(a[1, 0],
expm(a.generator())[1, 0],
places=ACCURATE_DIGITS,
msg=None,
delta=None)
self.assertAlmostEqual(a[1, 1],
expm(a.generator())[1, 1],
places=ACCURATE_DIGITS,
msg=None,
delta=None)
def incremental(self):
""" Create an incremental transition matrix set from a cumulative set
"""
if self.temporal_type is 'Incremental':
print("Transition Matrix Set is already incremental")
return
else:
val_set = []
periods = len(self.entries)
anm1 = self.entries[0]
val_set.append(anm1)
for k in range(1, periods):
an = self.entries[k]
anm1 = self.entries[k - 1]
anm1i = anm1.I
a = anm1i * an
a = tm.TransitionMatrix(a)
val_set.append(a)
self.entries = val_set
self.temporal_type = 'Incremental'
return
def remove(self, state, method):
""" Remove a transition matrix state and distribute its probability to other states according
to prescribed method
:param state: the state to remove
:type state: int
:returns: a transition matrix
"""
new_matrix = tm.TransitionMatrix(dimension=self.shape[0] - 1)
states = list(range(self.shape[0]))
del states[state]
# process all rows of the matrix except the state we remove
for i in states:
# probability to distribute
xp = self[i, state]
if 0.0 < xp < 1.0:
# process all columns of the matrix except the state we remove
w = xp / (1.0 - xp)
for j in states:
# weight of state among remaining states
new_matrix[i, j] = self[i, j] * (1.0 + w)
return new_matrix
"""
Examples using transitionMatrix to perform various transition matrix operations.
"""
import numpy as np
from scipy.linalg import expm
import transitionMatrix as tm
from transitionMatrix.creditratings.predefined import JLT, SP02NR
from transitionMatrix import dataset_path
print("> Initialize a 3x3 matrix with values")
A = tm.TransitionMatrix(values=[[0.6, 0.2, 0.2], [0.2, 0.6, 0.2], [0.2, 0.2, 0.6]])
print(A)
A.print_matrix(format_type='Standard', accuracy=2)
A.print_matrix(format_type='Percent', accuracy=1)
print("> Initialize a generic matrix of dimension n")
B = tm.TransitionMatrix(dimension=4)
print(B)
print("> Any list can be used for initialization (but not all shapes are valid transition matrices!)")
C = tm.TransitionMatrix(values=[1.0, 3.0])
print(C)
print("> Any numpy array can be used for initialization (but not all are valid transition matrices!)")
D = tm.TransitionMatrix(values=np.identity(5))
print(D)
"""
import transitionMatrix as tm
import numpy as np
from transitionMatrix import dataset_path
print("> Loading historical multi-period transitional matrices (cumulative mode) from csv file")
SnP_Set0 = tm.TransitionMatrixSet(csv_file=dataset_path + "sp_1981-2016.csv", temporal_type='Cumulative')
print("> Validate")
print(SnP_Set0.validate())
print(
"> We detect dimensionality problems. The matrices are not square (missing the trivial Default and NR transitions)")
print("> We must fix that to proceed. Augment matrices in set by fixing Default and NR transitions")
C_Vals = []
for matrix in SnP_Set0.entries:
C = tm.TransitionMatrix(values=np.resize(matrix, (9, 9)))
# set the migration from NR or D state to a rated state to zero
C[7, 0:9] = 0.0
C[8, 0:9] = 0.0
# set the probability of remaining to a D state to unity
C[7, 7] = 100.0
# set the probability of remaining to an NR state to unity
C[8, 8] = 100.0
C_Vals.append(C)
SnP_Set1 = tm.TransitionMatrixSet(values=C_Vals)
print("> Validate Again")
print(SnP_Set1.validate())
print("> Now we have square matrices but the format is not in probabilities!")
print("> Divide all entries by 100")
sequences = [[(0.0, 0), (0.5, 1), (1.0, 2)], [(0.0, 1), (0.3, 0), (0.8, 1)],
[(0.0, 2), (0.2, 1), (0.7, 2)]]
replication_count = 10
definition = [('0', "A"), ('1', "B"), ('2', "C")]
myState = tm.StateSpace(definition)
# myState = tm.StateSpace(definition)
input_data = dataset_generators.deterministic(sequences, replication_count)
print(input_data)
sorted_data = input_data.sort_values(['ID', 'Time'], ascending=[True, True])
cohort_data, cohort_bounds = tm.utils.bin_timestamps(sorted_data, cohorts=100)
print(80 * '=')
print(cohort_data)
myEstimator = es.CohortEstimator(states=myState,
cohort_bounds=cohort_bounds,
ci={
'method': 'goodman',
'alpha': 0.05
})
result = myEstimator.fit(cohort_data,
labels={
'Time': 'Time',
'State': 'State',
'ID': 'ID'
})
myMatrix = tm.TransitionMatrix(myEstimator.average_matrix)
myEstimator.print(select='Counts')
myMatrix.print_matrix(accuracy=3)
def test_minimal_matrix(self):
a = tm.TransitionMatrix(values=Minimal)
a.validate()
self.assertEqual(a.dimension, 3)
remove_stale=True)
print('Intervals : ', cohort_intervals)
print('> Transitions Summary Cohorted Data')
pp.pprint(transitions_summary(cohort_data))
myEstimator = CohortEstimator(states=myState,
cohort_bounds=cohort_intervals,
ci={
'method': 'goodman',
'alpha': 0.05
})
myEstimator.fit(cohort_data)
myMatrix = tm.TransitionMatrix(myEstimator.average_matrix, states=myState)
myMatrix.print_matrix(accuracy=3, format_type='Standard', labels=False)
myEstimator2 = AalenJohansenEstimator(states=myState)
labels = {'Time': 'Time', 'From': 'From', 'To': 'To', 'ID': 'ID'}
etm, times = myEstimator2.fit(canonical_data, labels=labels)
myMatrix2 = tm.TransitionMatrix(etm[:, :, -1])
G = myMatrix2.generator()
oneyear = tm.TransitionMatrix(expm(0.2 * G))
oneyear.print_matrix(accuracy=3)
def main():
print("Done")
# third-party software included in this distribution. You may not use this file except in
# compliance with the License.
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
""" Examples using transitionMatrix to perform operations with transition matrix sets sequences
"""
import transitionMatrix as tm
from transitionMatrix.creditratings.predefined import Generic as T1
print("-- Lets seed the set with a 3x3 matrix")
A = tm.TransitionMatrix(
values=[[0.6, 0.2, 0.2], [0.2, 0.6, 0.2], [0.2, 0.2, 0.6]])
print(A)
print("-- Identical future period transitions in incremental mode")
A_Set = tm.TransitionMatrixSet(values=A,
periods=3,
method='Copy',
temporal_type='Incremental')
print(A_Set.entries)
print(
"-- Identical future period transitions in cumulative mode using the power method"
)
B_Set = tm.TransitionMatrixSet(values=A,
periods=3,
method='Power',
def __init__(self,
dimension=2,
values=None,
periods=1,
temporal_type=None,
method=None,
json_file=None,
csv_file=None):
""" Create a new matrix set. Different options for initialization are:
* providing values as a list of list
* providing values as a numpy array
* loading from a csv file
* loading from a json file
Without data, a default identity matrix is generated with user specified dimension
:param values: initialization values
:param dimension: matrix dimensionality (default is 2)
:param method: matrix dimensionality (default is 2)
:param periods: List with the timesteps of matrix observations
:param temporal_type: matrix dimensionality (default is 2)
* Incremental: Each period matrix reflects transitions for that period
* Cumulative: Each period matrix reflects cumulative transitions from start to that period
:param json_file: a json file containing transition matrix data
:param csv_file: a csv file containing transition matrix data
:type values: list of lists or numpy array
:type dimension: int
:type temporal_type: str
:type json_file: str
:type csv_file: str
:returns: returns a TranstionMatrix object
:rtype: object
.. note:: The initialization in itself does not validate if the provided values form indeed a transition matrix set
:Example:
Instantiate a transition matrix set directly using a list of matrices
.. code-block:: python
C_Vals = [[[0.75, 0.25], [0.0, 1.0]], [[0.75, 0.25], [0.0, 1.0]]]
C_Set = tm.TransitionMatrixSet(values=C_Vals, temporal_type='Incremental')
"""
self.dimension = dimension
if values is not None:
# Copy a single matrix to all periods
if method is 'Copy':
val_set = []
for k in range(periods):
a = tm.TransitionMatrix(values)
val_set.append(a)
self.entries = val_set
self.temporal_type = 'Incremental'
self.periods = list(range(periods))
self.dimension = val_set[0].shape[0]
# Create a multi-period matrix assuming a Markov Chain
elif method is 'Power':
val_set = []
a = tm.TransitionMatrix(values)
val_set.append(a)
an = a
for k in range(periods - 1):
an = an * a
an = tm.TransitionMatrix(an)
val_set.append(an)
self.entries = val_set
self.temporal_type = 'Cumulative'
self.periods = list(range(periods))
self.dimension = val_set[0].shape[0]
# Use provided matrices as-is
elif method is None:
val_set = []
for entry in values:
a = tm.TransitionMatrix(entry)
val_set.append(a)
self.entries = val_set
self.temporal_type = temporal_type
self.periods = list(range(periods))
self.dimension = val_set[0].shape[0]
elif values is None and csv_file is not None:
# Initialize from file in csv format
# First row is meta data labels (From States, To States, Periods, Tenor List)
# Second row is meta data values (comma separated)
# Subsequent rows are Periods x Matrices in sequence
if not os.path.isfile(csv_file):
print("Input File Does not Exist")
exit()
f = open(csv_file)
header_dict = f.readline()
header_data = f.readline().split(',')
val_set = []
from_states = int(header_data.pop(0))
to_states = int(header_data.pop(0))
periods = int(header_data.pop(0))
tenors = [int(x) for x in header_data]
q = pd.read_csv(f, header=None, usecols=range(to_states))
for k in range(periods):
raw = q.iloc[k * from_states:(k + 1) * from_states]
a = tm.TransitionMatrix(raw.as_matrix())
val_set.append(a)
self.entries = val_set
self.temporal_type = temporal_type
self.periods = tenors
self.dimension = val_set[0].shape[0]
f.close()
elif values is None and json_file is not None:
# Initialize from file in json format
if not os.path.isfile(json_file):
print("Input File Does not Exist")
exit()
val_set = []
q = json.load(open(json_file))
periods = len(q)
for k in range(periods):
a = tm.TransitionMatrix(q[k])
val_set.append(a)
self.entries = val_set
self.temporal_type = temporal_type
self.periods = list(range(periods))
self.dimension = val_set[0].shape[0]
else:
# Default instance (2x2 identity matrix)
# default = np.identity(dimension)
val_set = []
for k in range(periods):
a = tm.TransitionMatrix(dimension=dimension)
val_set.append(a)
self.entries = val_set
if temporal_type is not None:
self.temporal_type = temporal_type
else:
self.temporal_type = 'Incremental'
self.periods = list(range(periods))
self.dimension = 2
self.validated = False
return
# Step 3
# Estimate matrices using Simple Estimator (Frequency count)
# compute confidence interval using goodman method at 95% confidence level
print("Step 3")
myEstimator = es.SimpleEstimator(states=myState,
ci={
'method': 'goodman',
'alpha': 0.05
})
# resulting matrix array is returned as result
result = myEstimator.fit(data)
# confidence levels are stored with the estimator
myEstimator.summary()
# Step 4
# Review numerical results
print("Step 4")
myMatrix = tm.TransitionMatrix(result)
myMatrix.print()
# In the LendingClub example we need to fix some matrix rows
# because there are no state_IN observations besides initial grade assignment
myMatrix[7, 9] = 1.0
myMatrix[8, 9] = 1.0
myMatrix[9, 9] = 1.0
print(myMatrix.validate())
print(myMatrix.characterize())
myMatrix.print()
Input data are the Standard and Poor's historical data (1981 - 2016) for corporate credit rating migrations
"""
import transitionMatrix as tm
from transitionMatrix import source_path
from transitionMatrix.creditratings.predefined import SP02, SP02NR
from transitionMatrix.utils import print_matrix
dataset_path = source_path + "datasets/"
example = 1
if example == 1:
a = tm.TransitionMatrix(values=SP02NR)
b = tm.TransitionMatrix(values=SP02)
a = 0.01 * a
b = 0.01 * b
a = a.remove(8, method='noninform')
print_matrix(a, format_type='Standard', accuracy=5)
print_matrix(b, format_type='Standard', accuracy=5)
elif example == 2:
print(
"> Load multi-period transitional matrices (cumulative mode) from json file"
)
SnP_Set0 = tm.TransitionMatrixSet(json_file=dataset_path +
"sp_1981-2016.json",
temporal_type='Cumulative')
"""
import matplotlib.pyplot as plt
from matplotlib import collections as mc
import transitionMatrix as tm
from transitionMatrix.predefined import Generic
from portfolioAnalytics.thresholds.model import ThresholdSet
from portfolioAnalytics.thresholds.settings import AR_Model
# Initialize a single period transition matrix
# Example 1: Generic -> Typical Credit Rating Transition Matrix
# Example 2: Minimal -> Three state transition matrix
M = tm.TransitionMatrix(values=Generic)
# The size of the rating scale
Ratings = M.dimension
# The Default (absorbing state)
Default = Ratings - 1
# Lets extend the matrix into multi periods
Periods = 10
T = tm.TransitionMatrixSet(values=M,
periods=Periods,
method='Power',
temporal_type='Cumulative')
# Initialize a threshold set
# compliance with the License.
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
# Example script. Open Risk Academy Course Step 1.
import transitionMatrix as tm
import numpy as np
from scipy.linalg import expm
from transitionMatrix.predefined import JLT
from transitionMatrix import dataset_path
A = tm.TransitionMatrix(
values=[[0.6, 0.2, 0.2], [0.2, 0.6, 0.2], [0.2, 0.2, 0.6]])
B = tm.TransitionMatrix(dimension=4)
C = tm.TransitionMatrix(values=[1.0, 3.0])
F = tm.TransitionMatrix(json_file=dataset_path + "JLT.json")
F.to_csv("JLT.csv")
print(A.validate())
print(B.validate())
print(C.validate())
print(F.validate())
C = tm.TransitionMatrix(values=np.resize(C, (2, 2)))
C[0, 1] = 0.0
C[1, 0] = 0.0
C[1, 1] = 1.0
my_colors = mymap(colors)
print(colors)
fig = plt.figure()
plt.style.use(['ggplot'])
plt.ylabel('Entity')
plt.xlabel('Time')
fig.suptitle('Entity Transitions Plot')
plt.scatter(x, y, marker='o', c=my_colors)
plt.margins(y=0.1, x=0.05)
plt.show()
elif example == 6:
filename = dataset_path + 'JLT.json'
print(filename)
myMatrix = tm.TransitionMatrix(json_file=filename)
myMatrix.print()
print(myMatrix.shape)
fig = plt.figure()
ax = fig.add_subplot(111, aspect='equal')
plt.style.use(['ggplot'])
plt.ylabel('From State')
plt.xlabel('To State')
mymap = plt.get_cmap("RdYlGn")
mymap = plt.get_cmap("Reds")
# mymap = plt.get_cmap("Greys")
normalize = mpl.colors.LogNorm(vmin=0.0001, vmax=1)
matrix_size = myMatrix.shape[0]
cohort_bounds = unique_timestamps(data)
# Estimate matrices using the Cohort estimator
myEstimator = CohortEstimator(states=myState,
cohort_bounds=cohort_bounds,
ci={
'method': 'goodman',
'alpha': 0.05
})
result = myEstimator.fit(data)
myMatrixSet = tm.TransitionMatrixSet(values=result,
temporal_type='Incremental')
myMatrixSet.cumulate()
myMatrixSet.print_matrix(period=8)
# Estimate matrices using the Aalen-Johansen estimator
canonical_data = to_canonical(data)
myEstimator2 = AalenJohansenEstimator(states=myState)
etm, times = myEstimator2.fit(canonical_data)
myMatrix2 = tm.TransitionMatrix(etm[:, :, -1])
print('Cumulative Empirical Matrix')
myMatrix2.print_matrix()
def main():
print("Done")
if __name__ == "__main__":
main()
