def test_smaller_root_size():
""" tests AlphaTree::ProcessAccess with rootSize = 256"""
# init AlphaTree
a = AlphaTree(256)
reuseDist = 0
memAddr = 0b11111111
a.ProcessAccess(memAddr, reuseDist)
sol = np.zeros((6, 2), dtype=np.float)
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b00000000
a.ProcessAccess(memAddr, reuseDist)
sol[5, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b11110000
a.ProcessAccess(memAddr, reuseDist)
# 5 non-reuse, 2-4 reuse, 1 non-reuse
sol[5, 0] += 1
sol[1, 0] += 1
sol[2:5, 1] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
def test_different_num_bins():
""" tests an alphaTree with a different number of reuse distance
bins"""
# init AlphaTree
a = AlphaTree(bins=7)
reuseDist = 0
memAddr = 0b111111111
a.ProcessAccess(memAddr, reuseDist)
sol = np.zeros((7, 7, 2), dtype=np.float)
# count should be all 0's
assert np.array_equal(a.reuseCount, sol)
reuseDist = 20
a.ProcessAccess(memAddr, reuseDist)
# all reused for rd >= 6
sol[6, :, 1] += 1
# count should be all 0's
assert np.array_equal(a.reuseCount, sol)
reuseDist = 4
a.ProcessAccess(memAddr, reuseDist)
# all reused for rd >= 6
sol[4, :, 1] += 1
# count should be all 0's
assert np.array_equal(a.reuseCount, sol)
def test_load_alphas():
""" tests AlphaTree::LoadAlphas to ensure values are stored correctly"""
# init AlphaTree
a = AlphaTree()
test_alphas = np.zeros((3, 7, 2), dtype=np.float)
test_alphas[:, :, 1] = np.arange(7) / 7.0
test_alphas[:, :, 0] = 1 - (np.arange(7) / 7.0)
a.LoadAlphas(test_alphas)
assert np.array_equal(test_alphas, a.reuseCount)
def test_tree_level():
""" tests AlphaTree::GetTreeLevel function"""
# init AlphaTree()
a = AlphaTree()
block = 512
# check level is top level
assert 6 == a.GetTreeLevel(block)
block = 8
assert 0 == a.GetTreeLevel(block)
block = 32
assert 2 == a.GetTreeLevel(block)
def test_normalize_reuse_count():
""" tests AlphaTree.NormalizeReuseCount to verify correct noralization"""
# init alphaTree
a = AlphaTree()
values = np.ones((3, 7, 2), dtype=np.float)
a.reuseCount = values
a.NormalizeReuseCount()
sol = np.zeros((3, 7, 2), dtype=np.float)
sol[:, :, :] = .5
assert np.array_equal(a.reuseCount, sol)
values = np.zeros((3, 7, 2), dtype=np.float)
a.reuseCount = values
a.NormalizeReuseCount()
sol = np.zeros((3, 7, 2), dtype=np.float)
sol[:, :, 1] = 1.0
assert np.array_equal(a.reuseCount, sol)
def test_different_reuse_distances():
""" tests the calculation of alpha values using different reuse
distances"""
# init default AlphaTree
a = AlphaTree()
reuseDist = 0
memAddr = 0b111111111
a.ProcessAccess(memAddr, reuseDist)
sol = np.zeros((3, 7, 2), dtype=np.float)
# count should be all 0's
assert np.array_equal(a.reuseCount, sol)
reuseDist = 1
a.ProcessAccess(memAddr, reuseDist)
sol[reuseDist, :, 1] += 1
assert np.array_equal(a.reuseCount, sol)
reuseDist = 2
a.ProcessAccess(memAddr, reuseDist)
sol[reuseDist, :, 1] += 1
assert np.array_equal(a.reuseCount, sol)
reuseDist = 0
memAddr = 0b011111111
a.ProcessAccess(memAddr, reuseDist)
# non-reuse for first node, no previous history for rest
sol[reuseDist, 6, 0] += 1
assert np.array_equal(a.reuseCount, sol)
def test_larger_root_size():
""" tests AlphaTree::ProcessAccess with rootSize = 1024"""
# init AlphaTree
a = AlphaTree(1024)
reuseDist = 0
memAddr = 0b1111111111
a.ProcessAccess(memAddr, reuseDist)
sol = np.zeros((8, 2), dtype=np.float)
assert np.array_equal(a.reuseCount[reuseDist], sol)
a.ProcessAccess(memAddr, reuseDist)
# all reused
sol[:, 1] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b0000000000
a.ProcessAccess(memAddr, reuseDist)
# one non-reuse at top
sol[7, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b1111100000
a.ProcessAccess(memAddr, reuseDist)
# 7 non-reuse, 3-6 reuse, 2 non-reuse, else nothing
sol[7, 0] += 1
sol[3:7, 1] += 1
sol[2, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
def GenerateApplicationProfile(traceFile,
outputFile,
reuseBins=3,
blockSize=512):
""" GenerateApplicationProfile: this function operates as the main routine
used to create an application profile from an input address & instruction
trace
args:
- traceFile: string indicating the name of the file that contains the
trace to be analyzed (plain-text)
- outputFile: string indicating the desired file name for the
application profile to be stored in (automatically append ".h5" to
filename)
- reuseBins: number of bins to group reuse distances into. For
reuseBins = K, the bins each correspond to a single reuse distance
except for index (K-1), which is for reuse distances >= K-1. Seperate
alpha values are calculated for each bin. Default value is 3
- blockSize: desired size of largest cache block to model. Default
is 512 bytes"""
# validate inputs
if reuseBins < 1:
raise ValueError("(in GenerateApplicationProfile) reuseBins >= 1")
if blockSize % 2 or blockSize < 8:
raise ValueError(
"(in GenerateApplicationProfile) blockSize must be power of 2 >= 8"
)
# set regular expression
regEx = re.compile("(\D),0x([0-9a-f]+)")
# set mask to pull blockAddress
blockMask = 2**32 - blockSize
# markov matrix for cycle activity
activityMarkov = np.zeros((2, 2), dtype=np.float)
previousCycle = 0 # indicates previous cycle's activity
# least-recently used ordered list of all 512-byte blocks accessed
lruStack = []
# probabilty mass function of all reuse distances
reusePMF = [0]
# maintains ordered vector of application's working set
workingSet = []
wsSize = 0
# list of load (read) proportions for each reuse distance
loadProp = [0]
lsMap = {'r': 0, 'w': 1}
# list of AlphaTree objects to collect alpha values
alphaForest = []
with open(traceFile) as file:
for line in file:
# process inactive cycle
if not re.search(regEx, line):
activityMarkov[previousCycle, 0] += 1
previousCycle = 0
continue
# process active cycle
activityMarkov[previousCycle, 1] += 1
previousCycle = 1
# seperate memory access into type and location
memAddress = int(re.search(regEx, line).group(2), 16)
accessType = re.search(regEx, line).group(1)
# get block address memAddress
memBlock = memAddress & blockMask
# convert access type to numeric representation
accessType = lsMap[accessType]
# look up reuse distance of this access
reuseDist = 0
while reuseDist < wsSize:
if lruStack[reuseDist] == memBlock:
break
reuseDist += 1
if reuseDist == wsSize: # if address not previously used
# process reuse distance
reusePMF[0] += 1
reusePMF.append(0)
# update lruStack
lruStack.insert(0, memBlock)
# increase size of loadProp to match reusePMF
loadProp.append(0)
if not accessType: # if load
loadProp[0] += 1
# allocate AlphaTree and process access
alphaForest.append(AlphaTree(blockSize, reuseBins))
alphaForest[wsSize - 1].ProcessAccess(memAddress, 0)
# add block to the working set
workingSet.append(memBlock)
wsSize += 1
continue
# process reuse distance
reusePMF[reuseDist + 1] += 1
# update lruStack
lruStack.remove(memBlock)
lruStack.insert(0, memBlock)
# process accesst type
if not accessType: # if load
loadProp[reuseDist + 1] += 1
# update appropriate AlphaTree
blockIndex = workingSet.index(memBlock)
alphaForest[blockIndex].ProcessAccess(memAddress, reuseDist)
# normalize load proprotions
for i in xrange(len(loadProp)):
if reusePMF[i]: # if non-zero
loadProp[i] /= float(reusePMF[i])
# normalize reuse PMF
reusePMF /= np.linalg.norm(reusePMF, 1)
# remove double counted inactive cycles
activityMarkov[0][0] -= activityMarkov[0][1]
# normalize activity markov model
activityMarkov[0][:] /= np.linalg.norm(activityMarkov[0][:], 1)
activityMarkov[1][:] /= np.linalg.norm(activityMarkov[1][:], 1)
# matrix to store calculated alpha values
alphas = np.zeros((wsSize, reuseBins, alphaForest[0].height, 2),
dtype=np.float)
# normalize and store alpha values
for i in xrange(wsSize):
# normalize count to get alpha values
alphaForest[i].NormalizeReuseCount()
# store in matrx
alphas[i] = alphaForest[i].reuseCount
""" structures stored in the profile:
- blockSize: input argument value
- workingSet: ordered list of the working set of the application
- reusePMF: probability mass function where the i-th index represents
the probability of reuse-distance = (i - 1) occuring. Index 0 indicates
the probability of a not-previously-accessed block being accessed
(reuse-distance = Inf). This also represents a compulsory cache miss
- loadProp: array where the i-th element corresponds to the probability
of a load (read) from memory occuring for reused-distance = (i-1).
Again, index 0 corresponds to prob(load) for a not-previously-accessed
block access. This is used to generate ld/str info for each access
based on the access' reuse distance
- activityMarkov: markov model for the probability of an inactive/active
memory cycle given whether the previous memory cycle was inactive/active.
0 corresponds to inactive, and 1 corresponds to active. Thus, the value
of activityMarkov[0][0] is the probability of an inactive cycle occuring
given the previous cycle was inactive
- alphas: matrix where the i-th row corresponds the the alpha values
for the ith block in workingSet. These are used to iteratively project
accesses to memory blocks into one half of the memory block based on
which half (aka subset) of the block was accessed previously. This
helps to model the spatial locality of the memory reference stream"""
# append 'h5' file extension
outputFile = outputFile + ".h5"
# save application profile to file
outputFile = h5.File(outputFile, 'w')
outputFile.create_dataset('blockSize', data=blockSize, dtype=np.int)
outputFile.create_dataset('workingSet',
data=np.asarray(workingSet, dtype=np.int))
outputFile.create_dataset('reusePMF',
data=np.asarray(reusePMF, dtype=np.float))
outputFile.create_dataset('loadProp',
data=np.asarray(loadProp, dtype=np.float))
outputFile.create_dataset('activityMarkov', data=activityMarkov)
outputFile.create_dataset('alphas', data=alphas)
outputFile.close()
def test_get_family():
""" tests node family functions for AlphaTree"""
# init AlphaTree
a = AlphaTree()
# start with root
nodeID = 0
# check left child ID
assert 1 == a.GetChild(nodeID, 0)
# check right child ID
assert 2 == a.GetChild(nodeID, 1)
nodeID = 1
# check parent is root
assert 0 == a.GetParent(nodeID)
# check sibling is 2
assert 2 == a.GetSibling(nodeID)
# check left child
assert 3 == a.GetChild(nodeID, 0)
# check right child
assert 4 == a.GetChild(nodeID, 1)
nodeID = 3
# check left child
assert 7 == a.GetChild(nodeID, 0)
# check right child
assert 8 == a.GetChild(nodeID, 1)
# check parent
assert 1 == a.GetParent(nodeID)
# check sibling is 4
assert 4 == a.GetSibling(nodeID)
def test_process_access():
""" tests AlphaTree::ProcessAccess function with sequence of memory accesses"""
# init default AlphaTree
a = AlphaTree()
reuseDist = 0
memAddr = 0b111111111
a.ProcessAccess(memAddr, reuseDist)
# count should be all 0's
assert np.array_equal(a.reuseCount[reuseDist],
np.zeros((7, 2), dtype=np.float))
a.ProcessAccess(memAddr, reuseDist)
sol = np.zeros((7, 2), dtype=np.float)
sol[:, 1] = 1.0
# should have every level reused
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b111110000
a.ProcessAccess(memAddr, reuseDist)
# top 5 reused, then 1 non-reuse & 1 nothing
sol[2:7, 1] += 1
sol[1, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b000000000
a.ProcessAccess(memAddr, reuseDist)
# should change except for first layer non-reuse
sol[6, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b000000100
a.ProcessAccess(memAddr, reuseDist)
# 6-1 reuse and 0 non-reuse
sol[1:7, 1] += 1
sol[0, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
memAddr = 0b111000000
a.ProcessAccess(memAddr, reuseDist)
# 6th non-reuse, 5-4 reuse, 3rd non-reuse, 2-0 nothing
sol[6, 0] += 1
sol[4:6, 1] += 1
sol[3, 0] += 1
assert np.array_equal(a.reuseCount[reuseDist], sol)
def test_generate_access():
""" test AlphaTree::GenerateAccess to ensure accesses are tracked
correctly and ouput is generated correctly based on the path
take when traversing the tree from root to leaf"""
# init AlphaTree
a = AlphaTree()
reuseDist = 0
# load alpha values
test_alphas = np.zeros((3, 7, 2), dtype=np.float)
test_alphas[:, :, 1] = np.arange(7) / 7.0
test_alphas[:, :, 0] = 1 - (np.arange(7) / 7.0)
a.LoadAlphas(test_alphas)
# test first access
test = a.GenerateAccess(reuseDist)
# traverse tree to find theoretical output
node = 0
block = 512
results = 0
while a.GetRightChild(node) < len(a.tree):
if a.tree[a.GetLeftChild(node)]:
node = a.GetLeftChild(node)
else:
node = a.GetRightChild(node)
results = results | (block >> 1)
block = block >> 1
# compare results
assert test == results
# test second access
test = a.GenerateAccess(reuseDist)
# traverse tree to find theoretical output
node = 0
block = 512
results = 0
while a.GetRightChild(node) < len(a.tree):
if a.tree[a.GetLeftChild(node)]:
node = a.GetLeftChild(node)
else:
node = a.GetRightChild(node)
results = results | (block >> 1)
block = block >> 1
# compare results
assert test == results
# load different distribution
test_alphas = np.ones((3, 7, 2), dtype=np.float)
test_alphas /= 2.0
a.LoadAlphas(test_alphas)
# test 3rd access
test = a.GenerateAccess(reuseDist)
# traverse tree to find theoretical output
node = 0
block = 512
results = 0
while a.GetRightChild(node) < len(a.tree):
if a.tree[a.GetLeftChild(node)]:
node = a.GetLeftChild(node)
else:
node = a.GetRightChild(node)
results = results | (block >> 1)
block = block >> 1
# compare results
assert test == results