url = urlRoot + 'series_id='+ series + '&api_key='+ apiKey + '&file_type=json'
resp = requests.get(url)
obs = resp.json()['observations']
dateList = [x['date'] for x in obs]
valList = [x['value'] for x in obs]
valueLabel = series + 'Value'
newDF = {'Date': dateList, valueLabel: valList}
newDF = pd.DataFrame(data = newDF)
newDF['Date'] = pd.to_datetime(newDF['Date'])
newDF.replace(['.'], np.nan, inplace = True)
newDF.fillna(method = 'ffill', inplace = True)
newDF[valueLabel] = newDF[valueLabel].astype('float64')
newDF[series + 'lastChangeP'] = (newDF[valueLabel] - newDF[valueLabel].shift(1)).div(newDF[valueLabel].shift(1))
fiveTickAverage = multiDayAnalysisTools.genXDayAverage(newDF, 5, valueLabel)
tenTickAverage = multiDayAnalysisTools.genXDayAverage(newDF, 10, valueLabel)
newDF[series + 'fiveVsTenTickAverage'] = (fiveTickAverage - tenTickAverage).div(tenTickAverage)
ticks = 20
newDF[series + 'bPercent'] = multiDayAnalysisTools.bBandPercent(newDF, ticks,valueLabel)
newDF = dateDF.merge(newDF, right_on = 'Date', left_on = 'Date', how = 'left')
seriesDFList.append(newDF)
os.chdir("data")
quandlRoot = 'quandlData/'
fileNames = ['AAII-AAII_SENTIMENT.csv', 'CASS-CFI.csv', 'CASS-CTLI.csv', 'USMISERY-INDEX.csv', 'WORLDAL-PALPROD.csv']
df = pd.read_csv(quandlRoot + fileNames[0], usecols = ['Date','Bull-Bear Spread', 'Neutral'], parse_dates = ['Date'])
df = dateDF.merge(df, right_on = 'Date', left_on = 'Date', how = 'left')
seriesDFList.append(df)
df = df.append(toAppend)
df.sort_values(by =['Date'], inplace = True, ascending = True)
df['titleSentChangeSinceYesterday'] = (df['titleSent'] - df['titleSent'].shift(1))
df['articleSentChangeSinceYesterday'] = (df['articleSent'] - df['articleSent'].shift(1))
df.to_csv(destFileName, index = False)
df = pd.read_csv(destFileName, dtype = {'titleSent': 'float64', 'titleSentChangeSinceYesterday': 'float64',
'articleSent': 'float64', 'articleSentChangeSinceYesterday': 'float64'}, parse_dates = ['Date'])
dateRange = np.arange('2013-01-01', '2020-05-26', dtype='datetime64[D]')
dateRange = {'Date': dateRange}
dateDF = pd.DataFrame(data = dateRange)
df['Date'] = pd.to_datetime(df['Date'])
newDF = dateDF.merge(df, right_on = 'Date', left_on = 'Date', how = 'left')
newDF.fillna(method = 'ffill', inplace = True)
days = 25
for x in ['titleSent', 'articleSent']:
Std = multiDayAnalysisTools.stdDevLastXDays(df, days, x)
Average = multiDayAnalysisTools.genXDayAverage(df, days, x)
upperBand = ((Std).multiply(2) + Average)
lowerBand = ((Std).multiply(-2) + Average)
bPercent = []
for close, upper, lower in zip (df[x], upperBand, lowerBand):
if upper - lower != 0:
bPercent.append((close - lower)/(upper - lower))
else:
bPercent.append(0)
df[x + 'bPercent'] = bPercent
newDF.to_csv(destFileName, index = False)
tenDayVolume.append(item)
if volCounter >= 10:
try:
volumeZList.append(
(item - (sum(tenDayVolume) / len(tenDayVolume))) /
statistics.stdev(tenDayVolume))
except:
volumeZList.append(0)
tenDayVolume.pop(0)
else:
volumeZList.append(np.nan)
df['VolumeZScoreTenDay'] = volumeZList
df['highVsLowPerc'] = ((df['High'] - df['Low']).div(df['Low']))
df['dayPercentChange'] = ((df['Close'] - df['Open']).div(df['Open']))
df['fiveDayAverage'] = multiDayAnalysisTools.genXDayAverage(
df, 5, 'Close')
df['tenDayAverage'] = multiDayAnalysisTools.genXDayAverage(
df, 10, 'Close')
df['fiveDayWeightedAverage'] = (((df['Close'] * 5) +
(df['Close'].shift(1) * 4) +
(df['Close'].shift(2) * 3) +
(df['Close'].shift(3) * 2) +
(df['Close'].shift(4))).div(15))
df['tenDayWeightedAverage'] = (
((df['Close'] * 10) + (df['Close'].shift(1) * 9) +
(df['Close'].shift(2) * 8) + (df['Close'].shift(3) * 7) +
(df['Close'].shift(4) * 6) + (df['Close'].shift(5) * 5) +
(df['Close'].shift(7) * 3) + (df['Close'].shift(8) * 2) +
(df['Close'].shift(9))).div(55))
df['fiveVSTenDayWeightedAverage'] = (df['fiveDayWeightedAverage'] -
df['tenDayWeightedAverage']).div(