def sequentialRunner():
# main parameters
sourceFileNumber = 1
performPCA = False # takes time, if false the code will use saved data from a previous run
utils.Constants(
).incrementalFunctionFit = True # linear sum of functions will fit functions incrementally if True, independantly if False
plotPCAResults = False # plots few charts using PCA results
utils.Constants().testType = utils.TEST_TYPE_RANDOM
utils.Constants().fractionFullSampleForTest = 0.5
utils.Constants().YVarianceBuckets = 14
# chose the file to process
sourceFile = getSourceFile(sourceFileNumber)
# derive the post PCA / regression file name
postLinearRegressionProcessingFile = getPCAProcessedFile(sourceFile)
# gets X values post PCA and prediction for all Ys
if performPCA:
# calls PCA / linear regression formula to prepare different sets of X0 to process and the prediction for each method
df = processRawFileWithLinearRegressions(
sourceFile, postLinearRegressionProcessingFile, steps=[1, 2, 3]
) # [1, 2, 3, 4]) step 4 takes time and proved not to work too well
else:
# recalls the last file saved post PCA
df = utils.parseFile(postLinearRegressionProcessingFile)
if plotPCAResults:
plotPCA(df)
classifiers.rateClassifiers(df)
def rateClassifier(df, Xlabels, XlabelsToNormalise, Ylabel, classifier):
dfToClassify = df[Xlabels + [Ylabel]].copy(deep=True)
dfToClassify = dfToClassify.replace([np.inf, -np.inf, 'inf'], 0)
df_filtered = utils.testNanInDF(dfToClassify, [Xlabels[0], Ylabel])
dfToClassify = dfToClassify.drop(df_filtered.index.values)
# normalize vectors which need to be
if XlabelsToNormalise:
dfToClassify[[
x[0] for x in XlabelsToNormalise
]] = StandardScaler(with_mean=True, with_std=True).fit_transform(
dfToClassify[[x[0] for x in XlabelsToNormalise
]]) * [x[1] for x in XlabelsToNormalise]
# separate training set and test set then fit the classifier and calculate accuracy
avgTestScore = 0
avgTrainScore = 0
for train, test in utils.getTestAndTrainSample().split(
dfToClassify[Xlabels]):
testClassifier = classifier()
testClassifier.fit(
dfToClassify.loc[train, Xlabels].values,
np.array(dfToClassify.loc[train, Ylabel].values, dtype=np.int64))
avgTrainScore += testClassifier.score(
dfToClassify.loc[train, Xlabels].values, dfToClassify.loc[
train, Ylabel].values) / utils.Constants().testSamplingRepeats
avgTestScore += testClassifier.score(
dfToClassify.loc[test, Xlabels].values, dfToClassify.loc[
test, Ylabel].values) / utils.Constants().testSamplingRepeats
return (round(avgTrainScore * 100, 3), round(avgTestScore * 100, 3))
def rateRest(df, ratings, classifier, restType):
"""
Assess (Y - predict) in three ways:
* discretized based on Y's standard deviation at stock level
* sign
* absolute discretized change
"""
nbVarianceBuckets = utils.Constants().YVarianceBuckets
nbBucketsPerVarianceUnit = utils.Constants().nbBucketsPerVarianceUnit
# variance groups
classificationLabelRest = utils.HEADER_Y_VARIANCE_GROUP + restType
df[classificationLabelRest] = modStats.ArrayFunctions.assignVarianceBasedGroup(
df[utils.HEADER_Y_REST + restType], df[utils.HEADER_Y_VARIANCE],
nbVarianceBuckets, nbBucketsPerVarianceUnit)
rateGroup(df, ratings, classifier, classificationLabelRest,
[utils.HEADER_DAY])
# sign
classificationLabelUpDown = utils.HEADER_Y_UPDOWN + restType
df[classificationLabelUpDown] = np.sign(df[utils.HEADER_Y_REST + restType])
rateGroup(df, ratings, classifier, classificationLabelUpDown,
[utils.HEADER_DAY])
# magnitude
classificationLabelMagnitude = utils.HEADER_Y_MAGNITUDE + restType
df[classificationLabelMagnitude] = df[classificationLabelRest].apply(abs)
rateGroup(df, ratings, classifier, classificationLabelMagnitude,
[utils.HEADER_DAY])
def rateClassifiers(df):
start = time()
ratings = dict()
# 50 shows some value improvement compared to 10 for acceptable performance using default KNN from sklearn kit
utils.Constants().kInKNN = 50
# .5 gives values a touch better for Y - PCA (X) p stock and does not change much others
utils.Constants().distDayWeight = 0.5
# rateDecisionTree(df, ratings)
# print(time() - start)
# print ('\n'.join([x + utils.SEPARATOR + str(ratings[x][0]) + utils.SEPARATOR + str(ratings[x][1]) for x in ratings.keys()]))
rateDefaultKNN(df, ratings)
print(time() - start)
print('\n'.join([
x + utils.SEPARATOR + str(ratings[x][0]) + utils.SEPARATOR +
str(ratings[x][1]) for x in ratings.keys()
]))
# rateCustomKNN(df, ratings)
# print(time() - start)
# print ('\n'.join([x + utils.SEPARATOR + str(ratings[x][0]) + utils.SEPARATOR + str(ratings[x][1]) for x in ratings.keys()]))
# rateDefaultLogisticRegression(df, ratings)
# print(time() - start)
print('\n'.join([
x + utils.SEPARATOR + str(ratings[x][0]) + utils.SEPARATOR +
str(ratings[x][1]) for x in ratings.keys()
]))
def testConstantsSetYVarianceBucketsSetTo21():
previousValue = utils.Constants().YVarianceBuckets
utils.Constants().YVarianceBuckets = 21
return (
utils.Constants().YVarianceBuckets == previousValue,
'Setting odd number for YVarianceBuckets should leave it at previous value.'
)
def rateYFromFile(df, ratings, classifier, Ylabel, labelSuffix):
"""
Assess any columns in the csv file in three ways:
* discretized based on Y's standard deviation at stock level
* sign
* absolute discretized change
"""
nbVarianceBuckets = utils.Constants().YVarianceBuckets
nbBucketsPerVarianceUnit = utils.Constants().nbBucketsPerVarianceUnit
# variance groups
classificationLabelYvariance = utils.HEADER_Y_VARIANCE_GROUP + labelSuffix
df[classificationLabelYvariance] = modStats.ArrayFunctions.assignVarianceBasedGroup(
df[utils.HEADER_Y + labelSuffix],
df[utils.HEADER_Y_VARIANCE + labelSuffix], nbVarianceBuckets,
nbBucketsPerVarianceUnit)
rateGroup(df, ratings, classifier, classificationLabelYvariance,
[utils.HEADER_DAY])
# sign
classificationLabel = utils.HEADER_Y_UPDOWN + labelSuffix
df[classificationLabel] = np.sign(df[Ylabel])
rateGroup(df, ratings, classifier, classificationLabel, [utils.HEADER_DAY])
# magnitude
classificationLabel = utils.HEADER_Y_MAGNITUDE + labelSuffix
nbVarianceBuckets = utils.Constants().YVarianceBuckets
df[classificationLabel] = df[classificationLabelYvariance].apply(abs)
rateGroup(df, ratings, classifier, classificationLabel, [utils.HEADER_DAY])
def processRawFileWithLinearRegressions(sourceFile,
targetFile,
fractionFullSampleForTest=0.2,
YVarianceBucketsParam=10,
steps=None):
utils.Constants().fractionFullSampleForTest = fractionFullSampleForTest
utils.Constants().YVarianceBuckets = YVarianceBucketsParam
logging.basicConfig(
filename=
'C:\\Users\\LL\\Desktop\\Work\\Machine learning\\challenge forecast markets\\log\\PCA.log',
level=logging.DEBUG,
format='%(asctime)s %(levelname)s:%(message)s')
df = utils.parseFile(sourceFile)
if steps == None:
steps = [1, 2, 3, 4]
if 1 in steps:
# simple PCA on the whole serie
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LINEAR, df, addRegressionVectors=True, addRest=False)
print(
'Linear predictor per market average coefficient of determination: '
+ str(coeffDetermin))
if 2 in steps:
# PCA on linear combination of functions on the whole serie
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LSUM_OF_FUNCTIONS,
df,
addRegressionVectors=True,
addRest=False)
print(
'Linear sum of functions predictor per market average coefficient of determination: '
+ str(coeffDetermin))
if 3 in steps:
# simple PCA on each stock independantly
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LINEAR_PER_STCK,
df,
addRegressionVectors=True,
addRest=True)
print(
'Linear predictor per stock average coefficient of determination: '
+ str(coeffDetermin))
if 4 in steps:
# PCA on linear combination of functions on each stock independantly
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LSUM_OF_FUNCTIONS_PER_STCK,
df,
addRegressionVectors=True,
addRest=True)
print(
'Linear sum of functions predictor per stock average coefficient of determination: '
+ str(coeffDetermin))
# write the new file with predicted data
df.to_csv(targetFile)
return df
def estimateRO(self, dfToPredict, Xlabels, Ylabel):
try:
transformedX = self.pca.transform(self.scaler.transform(dfToPredict[Xlabels]))
except ValueError as err:
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, err)
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, 'Vector below cannot be projected onto PCA vectors.')
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, dfToPredict[Xlabels])
return np.full((dfToPredict[Xlabels[0]].count(), 1), np.inf)
else:
return self.estimateROTransformedX(transformedX[:,:self.k])
def rateCustomKNN(df, ratings):
"""
KNN algo with customized distance _ weight for days and stocks can be adjusted
Implementing custom distances comes with two technical limitations:
* method is automatically switched to ball tree, which causes severe performance issues for large data sets without regularization
* the fit and predict functions have to be written in Python and not Cython _ not a major hurdle but good to notice
"""
utils.Constants().distDayWeight = 0
utils.Constants().distStockWeight = 0
utils.Constants().kInKNN = 10
rateOriginalY(df, ratings, getTunedKNNDistDayStockX)
def __init__(self):
self.c = utils.Constants()
self.test_input = tf.placeholder(tf.float32, shape=(1, 384, 384, 3))
self.sacc_module = tf.load_op_library(self.c.custom_lib_path)
self.sacc_result = self.sacc_module.sacc([self.test_input])
self.sesstion = tf.Session()
self.total_time = 0
def getBestLsumOfFunctionsPredictorTEST3(plot: False):
(HEADERS, DF, HEADER_X, HEADER_Y,
HEADER_WEIGHT) = utilsTest.testFromFile1()
utils.Constants().incrementalFunctionFit = True
myLsumOfFunctionsPredictor = modStats.mktPredictor.predictorFactory(
modStats.TYPE_LSUM_OF_FUNCTIONS,
DF,
HEADER_X,
HEADER_Y,
HEADER_WEIGHT,
utils.HEADER_MARKET,
utils.HEADER_STOCK,
displayCharts=plot)
estimate = myLsumOfFunctionsPredictor.estimate(utilsTest.forPrediction1(),
HEADER_X, HEADER_Y)
return (
np.array_equal(np.around(estimate, 6), [-0.085291, -0.900866])
and (round(myLsumOfFunctionsPredictor.getAvgCoeffDetermin(), 6)
== 0.945467)
and np.array_equal(
[f.__name__ for f in myLsumOfFunctionsPredictor.getXFunctions()], [
'log', 'identity', 'identity', 'digit80PercentOfAddRatio',
'digit90Percent', 'logOfMultRatio', 'identity', 'identity',
'identity', 'digit90Percent', 'log'
]),
'LsumOfFunctionsPredictor returns unexpected Value for prediction 1 from file 1 when functions are searched incrementally'
)
def main(**kwargs):
exp_const = utils.Constants()
exp_const.exp_dir = os.path.join(income_const['exp_dir'],
kwargs['exp_name'])
exp_const.log_dir = os.path.join(exp_const.exp_dir, 'logs')
exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models')
exp_const.lr = 1e-3
exp_const.weight_decay = 0
exp_const.num_epochs = 50
exp_const.batch_size = 256
exp_const.num_workers = 2
exp_const.gamma = 0.5
exp_const.loss = kwargs['loss']
exp_const.seed = 0
model_const = IncomeClassifierConstants()
model_const.num_hidden_blocks = kwargs['num_hidden_blocks']
datasets = {'train': FeatDataset('train'), 'val': FeatDataset('val')}
train_model(model_const, datasets, exp_const)
model_path = os.path.join(exp_const.model_dir, 'best_model')
print('Performance of best model selected during training ...')
state = torch.load(model_path)
print('Accuracy:\n\tTrain:', state['Accuracy']['train'], '\n\tVal:',
state['Accuracy']['val'])
print('Early stopping: \n\tEpoch:', state['Epoch'], '\n\tIter:',
state['Iter'], '\n\tStep:', state['Step'])
def get_data(dataset):
consts = utils.Constants(dataset)
test_spec = np.load('data/%s/%s.npz' % (dataset, 'test'))['spec']
test_Y = np.abs(test_spec[:, 0:1])
test_B = np.abs(test_spec[:, 1:2])
print('dataset shape', test_spec.shape)
return test_Y, test_B, test_spec, consts
def __init__(self, df, Xlabels, XFunctions, Ylabel, Wlabel):
dfTransformed = df.copy(deep = True)
self.XFunctions = XFunctions
if len(Xlabels) != len(XFunctions):
raise NameError('LsumOfFunctionsPredictor requires a function for each of X columns.')
for i in range(len(XFunctions)):
np.seterr(all='raise')
try:
dfTransformed[Xlabels[i]] = XFunctions[i](dfTransformed[Xlabels[i]])
except RuntimeWarning as e:
raise NameError('X values out of that set of functions definition range. Predictor cannot be built.')
except FloatingPointError as e:
raise NameError('X values out of that set of functions definition range. Predictor cannot be built.')
np.seterr(all='warn')
# instead of testing function definition, tests if the final result contains nan or +-inf
df_filtered = utils.testNanInDF(dfTransformed, Xlabels)
if not df_filtered.empty:
raise NameError('X values out of that set of functions definition range. Predictor cannot be built.')
try:
self.linearPredictor = LinearPredictor.getAccurateLinearPredictor(dfTransformed, Xlabels, Ylabel, Wlabel, False)
except NameError as err:
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, err)
raise NameError('X transformed by ' + '/'.join([f.__name__ for f in self.XFunctions]) + ' values cannot be fit by PCA.')
self.Xlabels = Xlabels
self.Ylabel = Ylabel
self.df = df
def getBestLsumOfFunctionsPredictor(df, Xlabels, Ylabel, Wlabel, displayCharts):
avgCoeffDetermins = []
bestavgCoeffDetermin = []
XFunctionsList = []
incrementalFit = utils.Constants().incrementalFunctionFit
# tries all functions for each vector _ assumption is that vetors are independant and functions should not be cross-checked
# a full check would have atrocious complexity
for Xlabel in Xlabels:
avgCoeffDeterminsForOneAxis = []
for f in LsumOfFunctionsPredictor.getFunctions():
avgCoeffDeterminsForOneAxis += [LsumOfFunctionsPredictor.testFitFunction(df, Xlabels, Xlabel, f, Ylabel, Wlabel, XFunctionsList, incrementalFit)]
# once all functions are tried, take the one minimizing the avgCoeffDetermin
sortedavgCoeffDeterminsForOneAxis = sorted(avgCoeffDeterminsForOneAxis, key=lambda x : x[0], reverse=True)
XFunctionsList += [sortedavgCoeffDeterminsForOneAxis[0][1]]
bestavgCoeffDetermin += [sortedavgCoeffDeterminsForOneAxis[0][0]]
avgCoeffDetermins += [avgCoeffDeterminsForOneAxis]
if displayCharts:
# print chosen functions
print([f.__name__ for f in XFunctionsList])
print(' / '.join([f.__name__ for f in LsumOfFunctionsPredictor.getFunctions()]))
print('\n'.join([' '.join([str(s) + ' /' for (s, f) in y]) for y in avgCoeffDetermins]))
fig, ax = plt.subplots()
for i, function in enumerate(LsumOfFunctionsPredictor.getFunctions()):
ax.plot(np.arange(len(Xlabels)), [l[i][0] for l in avgCoeffDetermins], label=function.__name__, marker=utils.getShapesToPlot(i), color=((109 - 15 * i)/255, (127 - 15 * i)/255, (255 - 15 * i)/255)) # plotting avgCoeffDetermins for each function separately
ax.plot(np.arange(len(Xlabels)), bestavgCoeffDetermin, 'ro-')
ax.legend(loc='upper center', shadow=True)
plt.show()
#returns the one giving the minimum
return LsumOfFunctionsPredictor(df, Xlabels, XFunctionsList, Ylabel, Wlabel)
def testFitFunction(df, Xlabels, Xlabel, function, Ylabel, Wlabel, prevFunctions, incrementalFit):
try:
# two options
if incrementalFit:
# tries all functions given all previous ones, to incrementally improve the result. Functions start from higher correl axis so their impact should have gradually less and less impact
predictor = LsumOfFunctionsPredictor(df, Xlabels, LsumOfFunctionsPredictor.getFunctionList(Xlabels, prevFunctions, function), Ylabel, Wlabel)
else:
# tries the functions one by one keepng all others to ID
predictor = LsumOfFunctionsPredictor(df, Xlabels, LsumOfFunctionsPredictor.getFunctionListForOneFunctionOneLabel(Xlabels, Xlabel, function), Ylabel, Wlabel)
result = (predictor.getAvgCoeffDetermin(), function)
del predictor
except NameError as err:
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, err)
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, function.__name__ + ' cannot be applied on the vector as it is not defined')
result = (np.NINF, function)
return result
def rateDefaultKNN(df, ratings):
"""
Using the standard KNN to predict discretised movement _ using each stock's variance _
K can be set to improve results _ turns up increasing K affects performance significantly with little improvement on the score
"""
classifierCreator = makeKNeighborsClassifier(utils.Constants().kInKNN)
# rateOriginalY(df, ratings, classifierCreator)
rateYIncrements(df, ratings, classifierCreator)
def __init__(self, df, Xlabels, groupHeaders, Ylabel, Wlabel):
self.segment = groupHeaders
grouped = df.groupby(groupHeaders)
self.basePredictors = dict()
self.avgWeightedSquareDifference = 0
self.XReduced = pd.DataFrame(np.full((df[Ylabel].count(), len(Xlabels)), np.inf), index = df.index.values)
# fit a linear estimator per group
missed = 0
for name, subdf in grouped:
try:
self.basePredictors[name] = self.__class__.buildBasePredictor(subdf.reset_index(drop=True), Xlabels, Ylabel, Wlabel)
self.avgWeightedSquareDifference += self.basePredictors[name].getAvgWeightedSquareDifference() * subdf[Ylabel].count()
self.XReduced.loc[subdf.index, range(self.basePredictors[name].getk())] = self.basePredictors[name].getX_reduced()
except NameError as err:
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, err)
logging.getLogger(utils.Constants().loggerName).log(logging.INFO, 'Stock ' + str(name) + ' cannot be regressed with PCA.')
missed += subdf[Ylabel].count()
if df[Ylabel].count() > missed:
self.avgWeightedSquareDifference = self.avgWeightedSquareDifference / (df[Ylabel].count() - missed)
self.avgCoeffDetermin = 1 - self.avgWeightedSquareDifference / sum((np.square(df[Ylabel]) - np.average(df[Ylabel]) ** 2) * df[Wlabel])
def get_data(dataset):
consts = utils.Constants(dataset)
train = np.load('data/%s/%s.npz' % (dataset, 'train'))['spec']
test_spec = np.load('data/%s/%s.npz' % (dataset, 'test'))['spec']
len_train = train.shape[0]
train_Y = np.abs(train[:len_train//2, 0:1])
train_B = np.abs(train[len_train//2:, 1:2])
train_B = train_B[:len(train_Y)]
test_Y = np.abs(test_spec[:,0:1])
test_B = np.abs(test_spec[:,1:2])
print ('dataset shape', train_Y.shape, train_B.shape, test_spec.shape)
return train_Y, train_B, test_Y, test_B, test_spec, consts
def SaccTest(self):
c = utils.Constants()
c.check()
expect_output = []
test_input = []
utils.load_test_input(test_input, c)
self.assertEqual(c.original_image_size, len(test_input))
utils.load_expect_output(expect_output, c)
sacc_module = tf.load_op_library(c.custom_lib_path)
with self.test_session():
engine = sacc_module.sacc(test_input)
result = engine.eval()
def rateGroup(df, ratings, classifier, classificationLabel,
additionalParameters):
"""
for any given target label vector and given classifier, fits the classifier based on
"""
ratings[createGroupName(
classifier.__name__, classificationLabel, utils.LABEL_X_ORIGINAL,
utils.Constants().distDayWeight)] = rateClassifier(
df, additionalParameters + utils.HEADER_X,
[(utils.HEADER_DAY, utils.Constants().distDayWeight)] +
[(x, 1) for x in utils.HEADER_X], classificationLabel, classifier)
ratings[createGroupName(
classifier.__name__, classificationLabel, utils.LABEL_X_LINEAR,
utils.Constants().distDayWeight)] = rateClassifier(
df, additionalParameters +
utils.getXVectors(df.columns.values, modStats.TYPE_LINEAR),
[(utils.HEADER_DAY, utils.Constants().distDayWeight)],
classificationLabel, classifier)
ratings[createGroupName(
classifier.__name__, classificationLabel,
utils.LABEL_X_LSUM_OF_FUNCTIONS,
utils.Constants().distDayWeight)] = rateClassifier(
df, additionalParameters + utils.getXVectors(
df.columns.values, modStats.TYPE_LSUM_OF_FUNCTIONS),
[(utils.HEADER_DAY, utils.Constants().distDayWeight)],
classificationLabel, classifier)
def main(**kwargs):
exp_const = utils.Constants()
exp_const.exp_dir = os.path.join(income_const['exp_dir'],
kwargs['exp_name'])
exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models')
exp_const.batch_size = 256
exp_const.num_workers = 1
model_const = IncomeClassifierConstants()
model_const.num_hidden_blocks = kwargs['num_hidden_blocks']
model_const.model_path = os.path.join(exp_const.model_dir, 'best_model')
dataset = FeatDataset('test')
eval_model(model_const, dataset, exp_const)
def getLsumOfFunctionsPerStockPredictorTEST1(plot: False):
(HEADERS, DF, HEADER_X, HEADER_Y,
HEADER_WEIGHT) = utilsTest.testFromFile726()
utils.Constants().incrementalFunctionFit = True
myLsumOfFunctionsPerStockPredictor = modStats.mktPredictor.predictorFactory(
modStats.TYPE_LSUM_OF_FUNCTIONS_PER_STCK, DF, HEADER_X, HEADER_Y,
HEADER_WEIGHT, utils.HEADER_MARKET, utils.HEADER_STOCK)
estimate = myLsumOfFunctionsPerStockPredictor.estimate(
utilsTest.forPrediction1(), HEADER_X, HEADER_Y)
return (
np.array_equal(np.around(estimate, 6), [-0.100358, -0.008756])
and (round(myLsumOfFunctionsPerStockPredictor.getAvgCoeffDetermin(), 6)
== 0.937514),
'LinearPerStockPredictor returns unexpected Value for prediction 1 from file 726'
)
def getLinearPerStockPredictorTEST0(plot: False):
utils.Constants().fractionFullSampleForTest = 0.2
(HEADERS, DF, HEADER_X, HEADER_Y,
HEADER_WEIGHT) = utilsTest.testFromFile726()
myLinearPerStockPredictor = modStats.mktPredictor.predictorFactory(
modStats.TYPE_LINEAR_PER_STCK, DF, HEADER_X, HEADER_Y, HEADER_WEIGHT,
utils.HEADER_MARKET, utils.HEADER_STOCK)
estimate = myLinearPerStockPredictor.estimate(utilsTest.forPrediction1(),
HEADER_X, HEADER_Y)
return (np.array_equal(np.around(estimate, 6), [
-0.850607, -1.915488
]) and (
round(myLinearPerStockPredictor.getAvgCoeffDetermin(), 6) == 0.847778
), 'LinearPerStockPredictor returns unexpected Value for prediction 1 from file 726'
)
def LinearPerStockPredictorGetX_reducedTEST0(plot: False):
utils.Constants().fractionFullSampleForTest = 0.2
(HEADERS, DF, HEADER_X, HEADER_Y,
HEADER_WEIGHT) = utilsTest.testFromFile726()
myLsumOfFunctionsPerStockPredictor = modStats.mktPredictor.predictorFactory(
modStats.TYPE_LSUM_OF_FUNCTIONS_PER_STCK, DF, HEADER_X, HEADER_Y,
HEADER_WEIGHT, utils.HEADER_MARKET, utils.HEADER_STOCK)
X_reduced = myLsumOfFunctionsPerStockPredictor.getX_reduced()
return (
(len(X_reduced) == 249) and (round(X_reduced[0, 0], 6) == -317.090575)
and (round(X_reduced[1, 1], 6) == -0.463664)
and (round(X_reduced[0, 9], 6) == 0.00056 and
(round(X_reduced[248, 0], 6) == 61.238643) and
(round(X_reduced[247, 8], 6) == -0.067226) and
(round(X_reduced[248, 9], 6) == -0.001823)),
'getX_reduced functions of LinearPerStockPredictor returns unexpected Value for file 726'
)
def separate_audio():
selected_song = playlistbox.curselection()
selected_song = int(selected_song[0])
play_it = playlist[selected_song]
consts = utils.Constants("speech")
model = torch.load('model-9', map_location='cpu').to(device)
audio = sf.read(play_it)
spec_abs, spec = utils.aud2spec(audio, consts)
test_mask = []
test_batch_Y = spec_abs
test_batch_Y = torch.from_numpy(test_batch_Y).to(device).float()
batch_mask = model(test_batch_Y)
test_mask.append(batch_mask.cpu().data.numpy())
test_mask = np.concatenate(test_mask, axis=0).squeeze()
B_test_prediction_spectrogram = spec[:, 0] * test_mask
X_test_prediction_spectrogram = spec[:, 0] * (1 - test_mask)
B_pred_audio = utils.spec2aud(
B_test_prediction_spectrogram.transpose((1, 0, 2)).reshape((257, -1)),
consts)
X_pred_audio = utils.spec2aud(
X_test_prediction_spectrogram.transpose((1, 0, 2)).reshape((257, -1)),
consts)
sf.write("B_test_pred.wav", B_pred_audio, consts.SR)
sf.write("X_test_pred.wav", X_pred_audio, consts.SR)
index = 0
playlistbox.insert(index, "B_test_pred")
playlist.insert(index, "B_test_pred.wav")
index += 1
playlistbox.insert(index, "X_test_pred")
playlist.insert(index, "X_test_pred.wav")
index += 1
def testTearDown():
utils.Constants().YVarianceBuckets = 42
utils.Constants().fractionFullSampleForTest = 0.22
utils.Constants().randomState == 42
utils.Constants().incrementalFunctionFit = True
utils.Constants().distDayWeight = 42
utils.Constants().testType == utils.TEST_TYPE_RANDOM
utils.Constants().distStockWeight = 33
utils.Constants().kInKNN = 7
utils.Constants.tearDown()
return ((utils.Constants().YVarianceBuckets == 22)
and (utils.Constants().fractionFullSampleForTest == 0.1)
and (utils.Constants().randomState == 12883823)
and (utils.Constants().incrementalFunctionFit == False)
and (utils.Constants().distDayWeight == 1)
and (utils.Constants().testType == utils.TEST_TYPE_K_FOLD)
and (utils.Constants().distStockWeight == 1000)
and (utils.Constants().kInKNN == 3),
'Constant tear down method does lead to param reset.')
def testConstantsKInKNNSetTo7():
utils.Constants().kInKNN = 7
return (utils.Constants().kInKNN == 7,
'Constant does not return the value set for distStockMajWeight.')
def testConstantsKInKNNDefault():
return (utils.Constants().kInKNN == 3,
'Default value given by Constant for kInKNN not as expected.')
def testConstantsDistStockWeightSetTo33():
utils.Constants().distStockWeight = 33
return (utils.Constants().distStockWeight == 33,
'Constant does not return the value set for distStockWeight.')
