def testSingleTS():
print("------------------- Test # 1 (Single TS). ------------------------")
p = 0.7
N = 50
M = 400
timeSteps = N * M
# train/test split
trainProp = 0.9
M1 = int(trainProp * M)
M2 = M - M1
trainPoints = N * M1
testPoints = N * M2
print("Generating data...")
harmonicsTS = harmonicDataTest(timeSteps)
trendTS = trendDataTest(timeSteps)
(armaTS, armaMeanTS) = armaDataTest(timeSteps)
meanTS = harmonicsTS + trendTS + armaMeanTS
combinedTS = harmonicsTS + trendTS + armaTS
# normalize the values to all lie within [-1, 1] -- helps with RMSE comparisons
# can use the tsUtils.unnormalize() function to convert everything back to the original range at the end, if needed
max1 = np.nanmax(combinedTS)
min1 = np.nanmin(combinedTS)
max2 = np.nanmax(meanTS)
min2 = np.nanmin(meanTS)
max = np.max([max1, max2])
min = np.min([min1, min2])
combinedTS = tsUtils.normalize(combinedTS, max, min)
meanTS = tsUtils.normalize(meanTS, max, min)
# produce timestamps
timestamps = np.arange('2017-09-10 20:30:00',
timeSteps,
dtype='datetime64[1m]') # arbitrary start date
# split the data
trainDataMaster = combinedTS[
0:
trainPoints] # need this as the true realized values for comparisons later
meanTrainData = meanTS[
0:
trainPoints] # this is only needed for various statistical comparisons later
# randomly hide training data: choose between randomly hiding entries or randomly hiding consecutive entries
(trainData,
pObservation) = tsUtils.randomlyHideValues(copy.deepcopy(trainDataMaster),
p)
# now further hide consecutive entries for a very small fraction of entries in the eventual training matrix
(trainData, pObservation) = tsUtils.randomlyHideConsecutiveEntries(
copy.deepcopy(trainData), 0.9, int(M1 * 0.25), M1)
# interpolating Nans with linear interpolation
# trainData = tsUtils.nanInterpolateHelper(trainData)
# test data and hidden truth
testData = combinedTS[-1 * testPoints:]
meanTestData = meanTS[
-1 *
testPoints:] # this is only needed for various statistical comparisons
# time stamps
trainTimestamps = timestamps[0:trainPoints]
testTimestamps = timestamps[-1 * testPoints:]
# once we have interpolated, pObservation should be set back to 1.0
pObservation = 1.0
# create pandas df
key1 = 't1'
trainMasterDF = pd.DataFrame(index=trainTimestamps,
data={key1: trainDataMaster
}) # needed for reference later
trainDF = pd.DataFrame(index=trainTimestamps, data={key1: trainData})
meanTrainDF = pd.DataFrame(index=trainTimestamps,
data={key1: meanTrainData})
testDF = pd.DataFrame(index=testTimestamps, data={key1: testData})
meanTestDF = pd.DataFrame(index=testTimestamps, data={key1: meanTestData})
# train the model
print("Training the model (imputing)...")
print('SVD')
nbrSingValuesToKeep = 5
mod = SVDModel(key1,
nbrSingValuesToKeep,
N,
M1,
probObservation=pObservation,
svdMethod='numpy',
otherSeriesKeysArray=[],
includePastDataOnly=True)
mod.fit(trainDF)
imputedDf = mod.denoisedDF()
print(" RMSE (training imputation vs mean) = %f" %
tsUtils.rmse(meanTrainDF[key1].values, imputedDf[key1].values))
print(" RMSE (training imputation vs obs) = %f" %
tsUtils.rmse(trainMasterDF[key1].values, imputedDf[key1].values))
return
print('ALS')
# uncomment below to run the ALS algorithm ; comment out the above line
mod = ALSModel(key1,
nbrSingValuesToKeep,
N,
M1,
probObservation=pObservation,
otherSeriesKeysArray=[],
includePastDataOnly=True)
mod.fit(trainDF)
# imputed + denoised data
imputedDf = mod.denoisedDF()
print(" RMSE (training imputation vs mean) = %f" %
tsUtils.rmse(meanTrainDF[key1].values, imputedDf[key1].values))
print(" RMSE (training imputation vs obs) = %f" %
tsUtils.rmse(trainMasterDF[key1].values, imputedDf[key1].values))
print("Forecasting (#points = %d)..." % len(testDF))
# test data is used for point-predictions
forecastArray = []
for i in range(0, len(testDF)):
pastPoints = np.zeros(N - 1) # need an N-1 length vector of past point
j = 0
if (i < N -
1): # the first prediction uses the end of the training data
while (j < N - 1 - i):
pastPoints[j] = trainMasterDF[key1].values[len(trainDF) -
(N - 1 - i) + j]
j += 1
if (j < N - 1): # use the new test data
pastPoints[j:] = testDF[key1].values[i - (N - 1) + j:i]
keyToSeriesDFNew = pd.DataFrame(data={key1: pastPoints})
prediction = mod.predict(pd.DataFrame(data={}),
keyToSeriesDFNew,
bypassChecks=False)
forecastArray.append(prediction)
print(" RMSE (prediction vs mean) = %f" %
tsUtils.rmse(meanTestDF[key1].values, forecastArray))
print(" RMSE (prediction vs obs) = %f" %
tsUtils.rmse(testDF[key1].values, forecastArray))
print("Plotting...")
plt.plot(np.concatenate((trainMasterDF[key1].values, testDF[key1].values),
axis=0),
color='gray',
label='Observed')
plt.plot(np.concatenate(
(meanTrainDF[key1].values, meanTestDF[key1].values), axis=0),
color='red',
label='True Means')
plt.plot(np.concatenate((imputedDf[key1].values, forecastArray), axis=0),
color='blue',
label='Forecasts')
plt.axvline(x=len(trainDF),
linewidth=1,
color='black',
label='Training End')
legend = plt.legend(loc='upper left', shadow=True)
plt.title('Single Time Series (ARMA + Periodic + Trend) - $p = %.2f$' % p)
plt.show()
trainingYears.append(str(i))
testYears = []
for i in range(yearTrainEnd, yearTestEnd, 1):
testYears.append(str(i))
trainDataMasterDict = {}
trainDataDict = {}
testDataDict = {}
for key in otherStates:
series = dfBasque[dfBasque["regionname"] == key]
trainDataMasterDict.update({key: series[trainingYears].values[0]})
# randomly hide training data
(trainData, pObservation) = tsUtils.randomlyHideValues(
copy.deepcopy(trainDataMasterDict[key]), p)
trainDataDict.update({key: trainData})
testDataDict.update({key: series[testYears].values[0]})
series = dfBasque[dfBasque["regionname"] == basqueKey]
trainDataMasterDict.update({basqueKey: series[trainingYears].values[0]})
trainDataDict.update({basqueKey: series[trainingYears].values[0]})
testDataDict.update({basqueKey: series[testYears].values[0]})
trainMasterDF = pd.DataFrame(data=trainDataMasterDict)
trainDF = pd.DataFrame(data=trainDataDict)
testDF = pd.DataFrame(data=testDataDict)
# model
rscModel = RobustSyntheticControl(
basqueKey,
def basque(filename):
# BASQUE COUNTRY STUDY
df = pd.read_csv(filename)
pivot = df.pivot_table(values='gdpcap', index='regionname', columns='year')
pivot = pivot.drop('Spain (Espana)')
dfBasque = pd.DataFrame(pivot.to_records())
allColumns = dfBasque.columns.values
states = list(np.unique(dfBasque['regionname']))
years = np.delete(allColumns, [0])
basqueKey = 'Basque Country (Pais Vasco)'
states.remove(basqueKey)
otherStates = states
yearStart = 1955
yearTrainEnd = 1971
yearTestEnd = 1998
singvals = 1
p = 0.8
trainingYears = []
for i in range(yearStart, yearTrainEnd, 1):
trainingYears.append(str(i))
testYears = []
for i in range(yearTrainEnd, yearTestEnd, 1):
testYears.append(str(i))
trainDataMasterDict = {}
trainDataDict = {}
testDataDict = {}
for key in otherStates:
series = dfBasque[dfBasque['regionname'] == key]
trainDataMasterDict.update({key: series[trainingYears].values[0]})
# randomly hide training data
(trainData, pObservation) = tsUtils.randomlyHideValues(copy.deepcopy(trainDataMasterDict[key]), p)
trainDataDict.update({key: trainData})
testDataDict.update({key: series[testYears].values[0]})
series = dfBasque[dfBasque['regionname'] == basqueKey]
trainDataMasterDict.update({basqueKey: series[trainingYears].values[0]})
trainDataDict.update({basqueKey: series[trainingYears].values[0]})
testDataDict.update({basqueKey: series[testYears].values[0]})
trainMasterDF = pd.DataFrame(data=trainDataMasterDict)
trainDF = pd.DataFrame(data=trainDataDict)
testDF = pd.DataFrame(data=testDataDict)
# model
rscModel = RobustSyntheticControl(basqueKey, singvals, len(trainDF), probObservation=1.0, modelType='als',
otherSeriesKeysArray=otherStates)
# fit the model
rscModel.fit(trainDF)
# save the denoised training data
denoisedDF = rscModel.model.denoisedDF()
# predict - all at once
predictions = rscModel.predict(testDF)
# plot
yearsToPlot = range(yearStart, yearTestEnd, 1)
interventionYear = yearTrainEnd - 1
plt.plot(yearsToPlot, np.append(trainMasterDF[basqueKey], testDF[basqueKey], axis=0), color='red',
label='observations')
plt.plot(yearsToPlot, np.append(denoisedDF[basqueKey], predictions, axis=0), color='blue', label='predictions')
plt.axvline(x=interventionYear, linewidth=1, color='black', label='Intervention')
# plt.ylim((-1, 0))
legend = plt.legend(loc='upper right', shadow=True)
plt.title('Abadie et al. Basque Country Case Study - $p = %.2f$' % p)
plt.show()
def testMultipleTS():
print(
"------------------- Test # 2 (Multiple TS). ------------------------")
p = 1.0
N = 50
M = 400
timeSteps = N * M
# train/test split
trainProp = 0.7
M1 = int(trainProp * M)
M2 = M - M1
trainPoints = N * M1
testPoints = N * M2
key1 = 't1'
key2 = 't2'
key3 = 't3'
otherkeys = [key2, key3]
includePastDataOnly = True
print("Generating data...")
harmonicsTS = harmonicDataTest(timeSteps)
trendTS = trendDataTest(timeSteps)
(armaTS, armaMeanTS) = armaDataTest(timeSteps)
meanTS = harmonicsTS + trendTS + armaMeanTS
combinedTS = harmonicsTS + trendTS + armaTS
combinedTS2 = (0.3 * combinedTS) + np.random.normal(
0.0, 0.5, len(combinedTS))
combinedTS3 = (-0.4 * combinedTS)
#normalize the values to all lie within [-1, 1] -- helps with RMSE comparisons
# can use the tsUtils.unnormalize() function to convert everything back to the original range at the end, if needed
max1 = np.nanmax([combinedTS, combinedTS2, combinedTS3])
min1 = np.nanmin([combinedTS, combinedTS2, combinedTS3])
max2 = np.nanmax(meanTS)
min2 = np.nanmin(meanTS)
max = np.max([max1, max2])
min = np.min([min1, min2])
combinedTS = tsUtils.normalize(combinedTS, max, min)
combinedTS2 = tsUtils.normalize(combinedTS2, max, min)
combinedTS3 = tsUtils.normalize(combinedTS3, max, min)
meanTS = tsUtils.normalize(meanTS, max, min)
# produce timestamps
timestamps = np.arange('2017-09-10 20:30:00',
timeSteps,
dtype='datetime64[1m]') # arbitrary start date
# split the data
trainDataMaster = combinedTS[
0:
trainPoints] # need this as the true realized values for comparisons later
trainDataMaster2 = combinedTS2[0:trainPoints]
trainDataMaster3 = combinedTS3[0:trainPoints]
meanTrainData = meanTS[
0:
trainPoints] # this is only needed for various statistical comparisons later
# randomly hide training data
(trainData,
pObservation) = tsUtils.randomlyHideValues(copy.deepcopy(trainDataMaster),
p)
(trainData2, pObservation) = tsUtils.randomlyHideValues(
copy.deepcopy(trainDataMaster2), p)
(trainData3, pObservation) = tsUtils.randomlyHideValues(
copy.deepcopy(trainDataMaster3), p)
# now further hide consecutive entries for a very small fraction of entries in the eventual training matrix
(trainData, pObservation) = tsUtils.randomlyHideConsecutiveEntries(
copy.deepcopy(trainData), 0.95, int(M1 * 0.25), M1)
(trainData2, pObservation) = tsUtils.randomlyHideConsecutiveEntries(
copy.deepcopy(trainData2), 0.95, int(M1 * 0.25), M1)
(trainData3, pObservation) = tsUtils.randomlyHideConsecutiveEntries(
copy.deepcopy(trainData3), 0.95, int(M1 * 0.25), M1)
# once we have interpolated, pObservation should be set back to 1.0
pObservation = 1.0
# interpolating Nans with linear interpolation
#trainData = tsUtils.nanInterpolateHelper(trainData)
#trainData2 = tsUtils.nanInterpolateHelper(trainData2)
#trainData3 = tsUtils.nanInterpolateHelper(trainData3)
# test data and hidden truth
testData = combinedTS[-1 * testPoints:]
testData2 = combinedTS2[-1 * testPoints:]
testData3 = combinedTS3[-1 * testPoints:]
meanTestData = meanTS[
-1 *
testPoints:] # this is only needed for various statistical comparisons
# time stamps
trainTimestamps = timestamps[0:trainPoints]
testTimestamps = timestamps[-1 * testPoints:]
# create pandas df
trainMasterDF = pd.DataFrame(index=trainTimestamps,
data={
key1: trainDataMaster,
key2: trainDataMaster2,
key3: trainDataMaster3
}) # needed for reference later
trainDF = pd.DataFrame(index=trainTimestamps,
data={
key1: trainData,
key2: trainData2,
key3: trainData3
})
meanTrainDF = pd.DataFrame(index=trainTimestamps,
data={key1: meanTrainData})
testDF = pd.DataFrame(index=testTimestamps,
data={
key1: testData,
key2: testData2,
key3: testData3
})
meanTestDF = pd.DataFrame(index=testTimestamps, data={key1: meanTestData})
# train the model
print("Training the model (imputing)...")
nbrSingValuesToKeep = 5
mod = SVDModel(key1,
nbrSingValuesToKeep,
N,
M1,
probObservation=pObservation,
svdMethod='numpy',
otherSeriesKeysArray=otherkeys,
includePastDataOnly=includePastDataOnly)
# uncomment below to run the ALS algorithm ; comment out the above line
#mod = ALSModel(key1, nbrSingValuesToKeep, N, M1, probObservation=pObservation, otherSeriesKeysArray=otherkeys, includePastDataOnly=True)
mod.fit(trainDF)
# imputed + denoised data
imputedDf = mod.denoisedDF()
print(" RMSE (training imputation vs mean) = %f" %
tsUtils.rmse(meanTrainDF[key1].values, imputedDf[key1].values))
print(" RMSE (training imputation vs obs) = %f" %
tsUtils.rmse(trainMasterDF[key1].values, imputedDf[key1].values))
print("Forecasting (#points = %d)..." % len(testDF))
# test data is used for point-predictions
otherTSPoints = N
if (includePastDataOnly == True):
otherTSPoints = N - 1
forecastArray = []
for i in range(0, len(testDF)):
pastPointsPrediction = np.zeros(
N - 1
) # for the time series of interest, we only use the past N - 1 points
# first fill in the time series of interest
j = 0
if (i < N -
1): # the first prediction uses the end of the training data
while (j < N - 1 - i):
pastPointsPrediction[j] = trainMasterDF[key1].values[
len(trainDF) - (N - 1 - i) + j]
j += 1
if (j < N - 1): # use the new test data
pastPointsPrediction[j:] = testDF[key1].values[i - (N - 1) + j:i]
# now fill in the other series
otherSeriesDataDict = {}
for key in otherkeys:
pastPointsOthers = np.zeros(
otherTSPoints
) # need an appropriate length vector of past points for each series
j = 0
if (i < N - 1
): # the first prediction uses the end of the training data
while (j < N - 1 - i):
pastPointsOthers[j] = trainMasterDF[key].values[
len(trainDF) - (N - 1 - i) + j]
j += 1
if (j < otherTSPoints): # use the new test data
if (includePastDataOnly == True):
pastPointsOthers[j:] = testDF[key].values[i - (N - 1) +
j:i]
else:
pastPointsOthers[j:] = testDF[key].values[i - (N - 1) +
j:i + 1]
otherSeriesDataDict.update({key: pastPointsOthers})
otherKeysToSeriesDFNew = pd.DataFrame(data=otherSeriesDataDict)
keyToSeriesDFNew = pd.DataFrame(data={key1: pastPointsPrediction})
prediction = mod.predict(otherKeysToSeriesDFNew,
keyToSeriesDFNew,
bypassChecks=False)
forecastArray.append(prediction)
print(" RMSE (prediction vs mean) = %f" %
tsUtils.rmse(meanTestDF[key1].values, forecastArray))
print(" RMSE (prediction vs obs) = %f" %
tsUtils.rmse(testDF[key1].values, forecastArray))
print("Plotting...")
plt.plot(np.concatenate((trainMasterDF[key1].values, testDF[key1].values),
axis=0),
color='gray',
label='Observed')
plt.plot(np.concatenate(
(meanTrainDF[key1].values, meanTestDF[key1].values), axis=0),
color='red',
label='True Means')
plt.plot(np.concatenate((imputedDf[key1].values, forecastArray), axis=0),
color='blue',
label='Forecasts')
plt.axvline(x=len(trainDF),
linewidth=1,
color='black',
label='Training End')
legend = plt.legend(loc='upper left', shadow=True)
plt.title('Single Time Series (ARMA + Periodic + Trend) - $p = %.2f$' % p)
plt.show()
def prop99(filename):
# CALIFORNIA PROP 99 STUDY
df = pd.read_csv(filename)
df = df[df['SubMeasureDesc'] == 'Cigarette Consumption (Pack Sales Per Capita)']
pivot = df.pivot_table(values='Data_Value', index='LocationDesc', columns=['Year'])
dfProp99 = pd.DataFrame(pivot.to_records())
allColumns = dfProp99.columns.values
states = list(np.unique(dfProp99['LocationDesc']))
years = np.delete(allColumns, [0])
caStateKey = 'California'
states.remove(caStateKey)
otherStates = states
yearStart = 1970
yearTrainEnd = 1989
yearTestEnd = 2015
singvals = 2
p = 1.0
trainingYears = []
for i in range(yearStart, yearTrainEnd, 1):
trainingYears.append(str(i))
testYears = []
for i in range(yearTrainEnd, yearTestEnd, 1):
testYears.append(str(i))
trainDataMasterDict = {}
trainDataDict = {}
testDataDict = {}
for key in otherStates:
series = dfProp99[dfProp99['LocationDesc'] == key]
trainDataMasterDict.update({key: series[trainingYears].values[0]})
# randomly hide training data
(trainData, pObservation) = tsUtils.randomlyHideValues(copy.deepcopy(trainDataMasterDict[key]), p)
trainDataDict.update({key: trainData})
testDataDict.update({key: series[testYears].values[0]})
series = dfProp99[dfProp99['LocationDesc'] == caStateKey]
trainDataMasterDict.update({caStateKey: series[trainingYears].values[0]})
trainDataDict.update({caStateKey: series[trainingYears].values[0]})
testDataDict.update({caStateKey: series[testYears].values[0]})
trainMasterDF = pd.DataFrame(data=trainDataMasterDict)
trainDF = pd.DataFrame(data=trainDataDict)
testDF = pd.DataFrame(data=testDataDict)
# model
rscModel = RobustSyntheticControl(caStateKey, singvals, len(trainDF), probObservation=1.0, modelType='als',
otherSeriesKeysArray=otherStates)
# fit the model
rscModel.fit(trainDF)
# save the denoised training data
denoisedDF = rscModel.model.denoisedDF()
# predict - all at once
predictions = rscModel.predict(testDF)
# plot
yearsToPlot = range(yearStart, yearTestEnd, 1)
interventionYear = yearTrainEnd - 1
plt.plot(yearsToPlot, np.append(trainMasterDF[caStateKey], testDF[caStateKey], axis=0), color='red',
label='observations')
plt.plot(yearsToPlot, np.append(denoisedDF[caStateKey], predictions, axis=0), color='blue', label='predictions')
plt.axvline(x=interventionYear, linewidth=1, color='black', label='Intervention')
legend = plt.legend(loc='lower left', shadow=True)
plt.title('Abadie et al. Prop 99 Case Study (CA) - $p = %.2f$' % p)
plt.show()