def test__smoothingGoodEnoughCheck(self):
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_softMarginAdditionalLoops=0,
test_weightsEpsilon=1.0,
test_smoothingEndCheckType='wgsum')
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata=smoothingMetadata)
smoothing.__setDictionary__(
smoothingMetadata=smoothingMetadata,
dictionary=self.model.getNNModelModule().named_parameters())
smoothing.countWeights = 1
a = 2.0
b = 1.5
ut.testCmpPandas(
smoothing._smoothingGoodEnoughCheck(
abs(a - b), smoothingMetadata=smoothingMetadata), 'bool',
False)
ut.testCmpPandas(
smoothing._smoothingGoodEnoughCheck(
abs(a - b), smoothingMetadata=smoothingMetadata), 'bool', True)
b = 3.1
ut.testCmpPandas(
smoothing._smoothingGoodEnoughCheck(
abs(a - b), smoothingMetadata=smoothingMetadata), 'bool',
False)
def test__sumWeightsToArrayStd(self):
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_weightIter=dc.DefaultWeightDecay(2),
test_smoothingEndCheckType='std',
test_epsilon=1.0,
test_weightsEpsilon=1.0,
test_hardEpsilon=1e-9,
test_softMarginAdditionalLoops=0,
test_lossContainer=3,
test_lossContainerDelayedStartAt=1,
test_weightsArraySize=2)
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata=smoothingMetadata)
smoothing.__setDictionary__(
smoothingMetadata=smoothingMetadata,
dictionary=self.model.getNNModelModule().named_parameters())
self.helperEpoch.epochNumber = 3
self.dataMetadata.epoch = 4
self.helper.loss = torch.Tensor([1.0])
smoothing(helperEpoch=self.helperEpoch,
helper=self.helper,
model=self.model,
dataMetadata=self.dataMetadata,
modelMetadata=None,
metadata=self.metadata,
smoothingMetadata=smoothingMetadata)
wg = smoothing.__getSmoothedWeights__(
smoothingMetadata=smoothingMetadata, metadata=self.metadata)
std = smoothing._sumWeightsToArrayStd(wg)
ut.testCmpPandas(std.item(), 'std', dc.ConfigClass.STD_NAN)
second_weights = {
'linear1.weight': [[11., 11., 11.]],
'linear1.bias': [13.],
'linear2.weight': [[11.], [11.], [11.]],
'linear2.bias': [13., 13., 13.]
}
self.model.setConstWeights(weight=11, bias=13)
smoothing(helperEpoch=self.helperEpoch,
helper=self.helper,
model=self.model,
dataMetadata=self.dataMetadata,
modelMetadata=None,
metadata=self.metadata,
smoothingMetadata=smoothingMetadata)
wg = smoothing.__getSmoothedWeights__(
smoothingMetadata=smoothingMetadata, metadata=self.metadata)
std = smoothing._sumWeightsToArrayStd(wg)
ut.testCmpPandas(
std.item(), 'std',
torch.std(
torch.Tensor([
(11 - 9.0) * 6 + (13 - 11.0) * 4,
(9.0 - 5) * 6 + (11.0 - 7) * 4
])).item()) # smoothed weights = saved weights -> 0
def test_calcMean(self):
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_weightIter=dc.DefaultWeightDecay(2),
test_smoothingEndCheckType='wgsum')
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata=smoothingMetadata)
smoothing.__setDictionary__(
smoothingMetadata=smoothingMetadata,
dictionary=self.model.getNNModelModule().named_parameters())
weights = dict(self.model.named_parameters())
self.compareDictToNumpy(iterator=weights, numpyDict=init_weights)
smoothing.calcMean(model=self.model,
smoothingMetadata=smoothingMetadata)
smoothedWg = smoothing.weightsArray.array
weights = dict(self.model.named_parameters())
i = smoothedWg[0]
self.compareDictToNumpy(iterator=weights, numpyDict=init_weights)
self.compareDictToNumpy(iterator=i, numpyDict=init_weights)
#########
second_weights = {
'linear1.weight': [[11., 11., 11.]],
'linear1.bias': [13.],
'linear2.weight': [[11.], [11.], [11.]],
'linear2.bias': [13., 13., 13.]
}
self.model.setConstWeights(weight=11, bias=13) # change model weights
smoothing.calcMean(model=self.model,
smoothingMetadata=smoothingMetadata)
weights = dict(self.model.named_parameters())
i = smoothedWg[1]
self.compareDictToNumpy(iterator=i, numpyDict=second_weights)
self.compareDictToNumpy(iterator=weights, numpyDict=second_weights)
########
third_weights = {
'linear1.weight': [[9., 9., 9.]],
'linear1.bias': [11.],
'linear2.weight': [[9.], [9.], [9.]],
'linear2.bias': [11., 11., 11.]
}
smoothing.countWeights = 2
sm_weights = smoothing.__getSmoothedWeights__(
smoothingMetadata=smoothingMetadata, metadata=None)
weights = dict(self.model.named_parameters())
self.compareDictToNumpy(iterator=sm_weights, numpyDict=third_weights)
self.compareDictToNumpy(iterator=weights, numpyDict=second_weights)
def test_experiment_weightedMean_MNIST_predefModel_alexnet(self):
with sf.test_mode():
modelName = "alexnet"
metadata = sf.Metadata(testFlag=True,
trainFlag=True,
debugInfo=True)
dataMetadata = dc.DefaultData_Metadata(
pin_memoryTest=True,
pin_memoryTrain=True,
epoch=1,
test_howOftenPrintTrain=2,
howOftenPrintTrain=3,
resizeTo=Test_RunExperiment.MNIST_RESIZE)
optimizerDataDict = {"learning_rate": 1e-3, "momentum": 0.9}
obj = models.alexnet()
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_weightIter=dc.DefaultWeightDecay(1.05),
test_device='cpu',
test_epsilon=1e-5,
test_hardEpsilon=1e-7,
test_weightsEpsilon=1e-6,
test_weightSumContainerSize=3,
test_weightSumContainerSizeStartAt=1,
test_lossContainer=20,
test_lossContainerDelayedStartAt=10)
modelMetadata = dc.DefaultModel_Metadata(
lossFuncDataDict={},
optimizerDataDict=optimizerDataDict,
device='cuda:0')
data = dc.DefaultDataMNIST(dataMetadata)
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata)
model = dc.DefaultModelPredef(obj=obj,
modelMetadata=modelMetadata,
name=modelName)
optimizer = optim.SGD(model.getNNModelModule().parameters(),
lr=optimizerDataDict['learning_rate'],
momentum=optimizerDataDict['momentum'])
loss_fn = nn.CrossEntropyLoss()
stat = dc.run(metadataObj=metadata,
data=data,
model=model,
smoothing=smoothing,
optimizer=optimizer,
lossFunc=loss_fn,
modelMetadata=modelMetadata,
dataMetadata=dataMetadata,
smoothingMetadata=smoothingMetadata)
def test__sumAllWeights(self):
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_smoothingEndCheckType='wgsum')
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata=smoothingMetadata)
smoothing.__setDictionary__(
smoothingMetadata=smoothingMetadata,
dictionary=self.model.getNNModelModule().named_parameters())
smoothing.countWeights = 1
smoothing.calcMean(model=self.model,
smoothingMetadata=smoothingMetadata)
sumWg = smoothing._sumAllWeights(smoothingMetadata=smoothingMetadata,
metadata=self.metadata)
ut.testCmpPandas(sumWg, 'weight_sum', 58.0)
if (__name__ == '__main__'):
#sf.StaticData.TEST_MODE = True
metadata = sf.Metadata(testFlag=True, trainFlag=True, debugInfo=True)
dataMetadata = dc.DefaultData_Metadata(pin_memoryTest=True,
pin_memoryTrain=True,
epoch=1,
fromGrayToRGB=False)
loop = 5
smoothingMetadata = None
if (sf.test_mode.isActive()):
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_weightIter=dc.DefaultWeightDecay(1.05),
test_epsilon=1e-5,
test_hardEpsilon=1e-7,
test_weightsEpsilon=1e-6)
else:
smoothingMetadata = dc.DefaultSmoothingOscilationWeightedMean_Metadata(
weightIter=dc.DefaultWeightDecay(1.05),
epsilon=1e-4,
hardEpsilon=1e-7,
weightsEpsilon=1e-6)
#####################
types = ('predefModel', 'CIFAR10', 'weightedMean')
try:
stats = []
rootFolder = sf.Output.getTimeStr() + ''.join(x + "_"
for x in types) + "set"
def test__getSmoothedWeights__(self):
smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_weightIter=dc.DefaultWeightDecay(2),
test_smoothingEndCheckType='wgsum',
test_epsilon=1.0,
test_weightsEpsilon=1.0,
test_hardEpsilon=1e-9,
test_softMarginAdditionalLoops=0,
test_lossContainer=3,
test_lossContainerDelayedStartAt=1,
test_weightsArraySize=2)
self.helperEpoch.maxTrainTotalNumber = 1000
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata=smoothingMetadata)
smoothing.__setDictionary__(
smoothingMetadata=smoothingMetadata,
dictionary=self.model.getNNModelModule().named_parameters())
self.helper.loss = torch.Tensor([1.0])
smoothing(helperEpoch=self.helperEpoch,
helper=self.helper,
model=self.model,
dataMetadata=self.dataMetadata,
modelMetadata=None,
metadata=self.metadata,
smoothingMetadata=smoothingMetadata)
self.compareDictToNumpy(
smoothing.__getSmoothedWeights__(
smoothingMetadata=smoothingMetadata, metadata=self.metadata),
{})
smoothing(helperEpoch=self.helperEpoch,
helper=self.helper,
model=self.model,
dataMetadata=self.dataMetadata,
modelMetadata=None,
metadata=self.metadata,
smoothingMetadata=smoothingMetadata) # aby zapisać wagi
self.compareDictToNumpy(
smoothing.__getSmoothedWeights__(
smoothingMetadata=smoothingMetadata, metadata=self.metadata),
init_weights)
self.model.setConstWeights(weight=17, bias=19)
w = (17 + 5 / 2) / 1.5
b = (19 + 7 / 2) / 1.5
self.checkSmoothedWeights(smoothing=smoothing,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothingMetadata=smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
w=w,
b=b)
self.model.setConstWeights(weight=23, bias=27)
w = (23 + 17 / 2) / 1.5
b = (27 + 19 / 2) / 1.5
self.checkSmoothedWeights(smoothing=smoothing,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothingMetadata=smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
w=w,
b=b)
self.model.setConstWeights(weight=31, bias=37)
w = (31 + 23 / 2) / 1.5
b = (37 + 27 / 2) / 1.5
self.checkSmoothedWeights(smoothing=smoothing,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothingMetadata=smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
w=w,
b=b)
def test__isSmoothingGoodEnough__(self):
self.smoothingMetadata = dc.Test_DefaultSmoothingOscilationWeightedMean_Metadata(
test_epsilon=1.0,
test_hardEpsilon=1e-9,
test_smoothingEndCheckType='wgsum',
test_weightsEpsilon=2.0,
test_softMarginAdditionalLoops=0,
test_lossContainer=3,
test_lossContainerDelayedStartAt=1,
test_weightsArraySize=3)
smoothing = dc.DefaultSmoothingOscilationWeightedMean(
smoothingMetadata=self.smoothingMetadata)
smoothing.__setDictionary__(
smoothingMetadata=self.smoothingMetadata,
dictionary=self.model.getNNModelModule().named_parameters())
self.helper.loss = torch.Tensor([1.0])
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=1.0,
avgKLoss=0,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.helper.loss = torch.Tensor([0.5])
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=1.5 / 2,
avgKLoss=1.0,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=2 / 3,
avgKLoss=1.5 / 2,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.helper.loss = torch.Tensor([1.5])
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=2.5 / 3,
avgKLoss=1 / 2,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=3.5 / 3,
avgKLoss=2 / 2,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=4.5 / 3,
avgKLoss=3 / 2,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.helper.loss = torch.Tensor([1.3])
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=4.3 / 3,
avgKLoss=3 / 2,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=False)
self.checkOscilation__isSmoothingGoodEnough__(
avgLoss=4.1 / 3,
avgKLoss=2.8 / 2,
helperEpoch=self.helperEpoch,
dataMetadata=self.dataMetadata,
smoothing=smoothing,
smoothingMetadata=self.smoothingMetadata,
helper=self.helper,
model=self.model,
metadata=self.metadata,
booleanIsGood=True)