def load_data_1(stock, y, window): raw_data = stock.as_matrix() length = raw_data.shape[0] indicators = raw_data.shape[1] prices = y.as_matrix() data = [] for index in range(len(raw_data) - (window) + 1): data.append(raw_data[index:index + window]) data = np.array(data) valid_size = int(np.round(2 / 100 * data.shape[0])) test_size = int(1) training_size = data.shape[0] - (valid_size + test_size) X_train = data[:training_size + valid_size, :-1] y_train = prices[window - 1:training_size + window + valid_size - 1] #validation for time series X_valid = data[training_size:training_size + valid_size, :-1] y_valid = prices[training_size + window - 1:training_size + valid_size + window - 1] raw_data = raw_data.reshape(1, length, indicators) X_test = np.zeros((1, window, 8)) for i in range(0, window - 1): X_test[0, window - 1 - i, :] = raw_data[0, length - i - 1, :] X_test = X_test[:, 1:, :] y_test = 0 return [X_train, y_train, X_valid, y_valid, X_test, y_test]
def __plot_backtest(self, performance, performanceBenchmark, composition, names): # PERFORMANCE performance.index = performance.index.date df_to_plot = pd.concat([performance, performanceBenchmark], axis=1) fig = px.line(df_to_plot, x=df_to_plot.index, y=df_to_plot.columns, title='Comparison of different strategies') fig.show() # COMPOSITION composition.columns = list(names) composition = composition.loc[:, (composition != 0).any(axis=0)] data = [] for isin in composition.columns: trace = go.Bar( x=composition.index, y=composition[isin], name=str(isin) ) data.append(trace) layout = go.Layout(barmode='stack') fig = go.Figure(data=data, layout=layout) fig.update_layout( title="Portfolio Composition", xaxis_title="Number of the Investment Period", yaxis_title="Composition", legend_title="Name of the Fund") fig.layout.yaxis.tickformat = ',.1%' fig.show()
def get_unrolls(self): data, label = [], [] for i in range(self.config.num_unrollings): d, l = self.next_batch() data.append(d) label.append(l) return data, label
def get_sp100(): data = [] r = requests.get('https://en.wikipedia.org/wiki/S%26P_100') soup = BeautifulSoup(r.content,'html.parser') table = soup.find('table', attrs={'id':'constituents'}) table_body = table.find('tbody') rows = table_body.find_all('tr') for row in rows: cols = row.find_all('td') cols = [ele.text.strip() for ele in cols] if cols!=[]: cols = cols[0] data.append(cols) else: pass return data
def compound_data(request): cost = float(request.GET['cost']) change_ratio = int(request.GET['change_ratio']) ratio = change_ratio / float(100) count = int(request.GET['count']) data = list() for i in range(count): c = cost * ratio if random.choice((True, False)): cost += c cc = '+' + dots(c) else: cost -= c cc = '-' + dots(c) data.append([cost,dots(cost), cc, ]) return HttpResponse(json.dumps(data), content_type='text/json')
def get_historical_data(name, number_of_days): data = [] url = "https://finance.yahoo.com/quote/" + name + "/history/" rows = bs(requests.get(url, verify = True).text, features = "lxml").findAll('table')[0].tbody.findAll('tr') for each_row in rows: divs = each_row.findAll('td') if divs[1].span.text != 'Divident': data.append({ 'Date': divs[0].span.text, 'Open': float(divs[1].span.text.replace(',','')), 'Close': float(divs[4].span.text.replace(',','')), 'High': float(divs[2].span.text.replace(',','')), 'Low': float(divs[3].span.text.replace(',','')), 'Volume': float(divs[6].span.text.replace(',','')), 'Adj_Closing': float(divs[5].span.text.replace(',',''))}) return data[:number_of_days]
def _get_table_data_list(self): ''' A function for fetching NSE stock information Input: Url of the index page Outout: Header and Data list ''' url = self.url with urllib.request.urlopen(url) as response: html = BeautifulSoup(response.read(), 'html.parser') table = html.find('table', attrs={'class': 'tbldata14 bdrtpg'}) header = [] data = [] for row in table.findAll('tr'): # find header row if len(header) < 1: for col in row.findAll('th'): col_b = col.find('b') col_br = col.find('br') if col_b: header.append(col_b.string) elif (col_br): header.append('{0} {1}'.format( col_br.previous_sibling.string, col_br.string ) if col_br.string else col_br.previous_sibling) else: header.append(col.string) row_list = [] for col in row.findAll('td'): col_b = col.find('b') if col_b: row_list.append(col_b.string) href = col.find(href=re.compile('stockpricequote')) if href: row_list.append(href.get('href')) else: row_list.append(col.string) if len(row_list) > 1: data.append(row_list) header.insert(1, 'Link') return header, data
def get(self, request, pk, format=None): try: watch = Watch.objects.get(watchId=pk) wSerializer = WatchSerializer(watch) watch = wSerializer.data data = [] user = User.objects.get(userId=watch['userId']) userSerializer = UserSerializer(user) if watch['watchId'] > 0: watchassets = WatchAsset.objects.all().filter( watchId=watch['watchId']) wASerializer = WatchAssetSerializer(watchassets, many=True) for wAsset in wASerializer.data: if wAsset['assetId'] > 0: assets = Asset.objects.all().filter( assetId=wAsset['assetId']) aSer = AssetSerializer(assets, many=True) for key in aSer.data: item = { 'watchId': watch['watchId'], 'watchName': watch['watchName'], 'assetId': wAsset['assetId'], "assetName": key['assetName'], "volume": key['volume'], "open": key['open'], "close": key['close'], "low": key['low'], "high": key['high'], "adjClose": key['adjClose'], "date": key['date'], "userId": userSerializer.data['userId'], "name": userSerializer.data['name'], "email": userSerializer.data['email'] } data.append(item) return Response(data, status=status.HTTP_200_OK) except (RuntimeError, TypeError, NameError): data = {"description": "Object Not Found"} return Response(data, status=status.HTTP_404_NOT_FOUND)
def update_in_db(df, table_name, creds): matrix = np.array(df.to_records().view(type=np.matrix))[0] data = [] for i in range(len(matrix)): conv_date = pd.to_datetime(matrix[i][0]) date = "('" + str(conv_date.year) + "-" + str( conv_date.month) + "-" + str(conv_date.day) + "')::date" High = str(matrix[i][1]) Low = str(matrix[i][2]) Open = str(matrix[i][3]) Close = str(matrix[i][4]) Volume = str(matrix[i][5]) Adj_Clos = str(matrix[i][6]) prices = "(" + date + ", " + High + ", " + Low + ", " + Open + ", " + Close + ", " + Volume + "," + Adj_Clos + ")" data.append(prices) print(data) data = str(data).replace("[", "(").replace("]", ")").replace( '(', '', 1)[:-1].replace('"', '') table_name = table_name.replace('-', '_') query = """INSERT INTO {} (date, high, low, open, close, volume, adj_close) VALUES {} ON CONFLICT ON CONSTRAINT {}_date_key DO NOTHING;""".format( table_name.upper(), data, table_name.lower()) if i % 10000 == 0: try: conn = db_connection(creds) download_data(conn, query) data = [] logging.info("Se guardó: {}".format(table_name)) except Exception as error: logging.error("Error al tratar de insertar %s: %s" % (table_name, error)) elif i == len(matrix) - 1: try: conn = db_connection(creds) download_data(conn, query) logging.info("Se guardó: {}".format(table_name)) except Exception as error: logging.error("Error al tratar de insertar %s: %s" % (table_name, error))
def update_src_image(selector): # print('two') data = [] data.append({ 'x': pltdt.Date, 'y': pltdt.Close, 'type': 'line', 'name': 'Close' }) if 'Bollinger Bands' in selector: data = get_mdt_bb(df, data, stats_mdt) else: res = stats_mdt.pop('Bollinger Bands', None) if 'Donchian Channels' in selector: data = get_mdt_dc(df, data, stats_mdt) else: res = stats_mdt.pop('Donchian Channels', None) if 'Double Exponential Moving Avearge' in selector: data = get_mdt_dema(df, data, stats_mdt) else: res = stats_mdt.pop('Double Exponential Moving Avearge', None) if 'Hull MA' in selector: data = get_mdt_hma(df, data, stats_mdt) else: res = stats_mdt.pop('Hull MA', None) if 'Ichimoku Cloud' in selector: data = get_mdt_ic(df, data, stats_mdt) else: res = stats_mdt.pop('Ichimoku Cloud', None) if 'Keltner Channels' in selector: data = get_mdt_kc(df, data, stats_mdt) else: res = stats_mdt.pop('Keltner Channels', None) if 'MA Cross' in selector: data = get_mdt_macross(df, data, stats_mdt) else: res = stats_mdt.pop('MA Cross', None) if 'Moving Average Exponential' in selector: data = get_mdt_ema(df, data, stats_mdt) else: res = stats_mdt.pop('Moving Average Exponential', None) if 'Moving Average Simple' in selector: data = get_mdt_sma(df, data, stats_mdt) else: res = stats_mdt.pop('Moving Average Simple', None) if 'Smoothed Moving Average' in selector: data = get_mdt_smma(df, data, stats_mdt) else: res = stats_mdt.pop('Smoothed Moving Average', None) if 'Triple EMA' in selector: data = get_mdt_tema(df, data, stats_mdt) else: res = stats_mdt.pop('Triple EMA', None) if 'Moving Average Weighted' in selector: data = get_mdt_wma(df, data, stats_mdt) else: res = stats_mdt.pop('Moving Average Weighted', None) # print(stats_mdt) figure = { 'data': data, 'layout': { #'height': 300 } } return figure
pltdt = pd.DataFrame() pltdt['Date'] = df['Date'] pltdt['Close'] = df['Close'] # buySellData = [] initial = [] globalArray = [] stats_mdt = {} globalStats = {} globalbuysellSlectedValue = 0 data = [] data.append({ 'x': pltdt.Date, 'y': pltdt.Close, 'type': 'line', 'name': 'Close' }) def get_results(*args): print('Arguments for get_result:') print(args) print(stats_mdt) spv = '' if args[0] == 'All': # Run for indicator combination. sargslist = [] master = args[3] sargslist.append(stats_mdt[master]) # Add the master key first. for key in stats_mdt.keys():
def getWordCloud(q1): text = "Insert Your text here" try: newsapi = NewsApiClient(api_key='Your Key') except: newsapi = NewsApiClient(api_key='Your another Key incase of exception') end = date.today() start = date.today() - relativedelta(days=+10) s1 = str(start).split(" ")[0] e1 = str(end).split(" ")[0] try: news = newsapi.get_everything(q=q1, from_param=s1, to=e1, language='en', sort_by='popularity', page_size=100, page=1) #arranging the news articles in a numpy array news_data = [ ] #np.array(['publishedAt','title','description','content','url']) index = 0 for i in news['articles']: k = 0 data = [] while k != 1: data.append(i["publishedAt"]) data.append(i["title"]) data.append(i["description"]) data.append(i["content"]) data.append(i["url"]) data.append(i["urlToImage"]) k = 1 news_data.append(data) nd = np.array(news_data) data = pd.DataFrame(nd, columns=[ 'Date', 'Title', 'Description', 'Content', 'URL', 'Image' ]) #Spliting the date and time of each data field. new = data["Date"].str.split("T", n=1, expand=True) new1 = new[1].str.split("Z", n=1, expand=True) data = data.drop("Date", axis=1) data.insert(loc=0, column='Date', value=new[0]) data.insert(loc=1, column='Time', value=new1[0]) data['Date'] = pd.to_datetime(data.Date) dff = data[['Date', 'Description']].groupby('Date').sum() text = " ".join(list(dff['Description'])) + q1 * 15 wave_mask = np.array(Image.open("assets/cloud.jpg")) wordcloud = WordCloud(mask=wave_mask, width=512, height=512, colormap="Greens").generate(text) in1 = str(random.randint(0, 100000000000000)) in2 = str(random.randint(0, 100000000000000)) in3 = str(random.randint(0, 100000000000000)) in4 = in1 + in2 + in3 shutil.rmtree('assets/images') os.mkdir("assets/images") # Try wordcloud.to_file("assets/images/" + in4 + ".jpg") print("saved...") ## img1 = base64.b64encode(open("assets/images/"+in4+".jpg", 'rb').read()) with open("assets/images/" + in4 + ".jpg", "rb") as imageFile: img1 = base64.b64encode(imageFile.read()).decode("utf-8") print("Yes\n\n") return [img1] except Exception as e: print(e) try: newsapi = NewsApiClient(api_key='Your another key') news = newsapi.get_everything(q=q1, from_param=s1, to=e1, language='en', sort_by='popularity', page_size=100, page=1) print(news, "1") #arranging the news articles in a numpy array news_data = [ ] #np.array(['publishedAt','title','description','content','url']) index = 0 for i in news['articles']: k = 0 data = [] while k != 1: data.append(i["publishedAt"]) data.append(i["title"]) data.append(i["description"]) data.append(i["content"]) data.append(i["url"]) data.append(i["urlToImage"]) k = 1 news_data.append(data) nd = np.array(news_data) data = pd.DataFrame(nd, columns=[ 'Date', 'Title', 'Description', 'Content', 'URL', 'Image' ]) #Spliting the date and time of each data field. new = data["Date"].str.split("T", n=1, expand=True) new1 = new[1].str.split("Z", n=1, expand=True) data = data.drop("Date", axis=1) data.insert(loc=0, column='Date', value=new[0]) data.insert(loc=1, column='Time', value=new1[0]) data['Date'] = pd.to_datetime(data.Date) ## data = data.sort_values(by='Date') data = data.sort_values(by=['Date'], ascending=False) return data except: return [] return []
#df["Status"] = [tesla.inc_dec(c, o) for c, o in zip(df.Close, df.Open)] #df["Middle"] = (df.Open + df.Close) / 2 df["Daily Change"] = abs(df.Close - df.Open) date = df.index.values openprice = df['Open'] closeprice = df['Close'] high = df['High'] low = df['Low'] ## Get data for candlestick_ochl() and append to a list ## x = 0 y = len(date) data = [] while x < y: append_me = date[x], openprice[x], closeprice[x], high[x], low[x] data.append(append_me) x += 1 print(type(date[1])) ## Plot Candlesticks ## fig, ax1 = pyplot.subplots() candlestick_ochl(ax1, data, width=np.timedelta64(12, 'h'), colorup='green', colordown='red', alpha=0.75) ## Plot 20-Day SMA overlapping Candlestick plot ## ax1.plot(df['20-day SMA'], color='#26baee', linestyle='--') ## Plot Upper Bands ## ax1.plot(df['Upper Band'], color='#783eff', linestyle='-') ## Plot Lower Bands ## ax1.plot(df['Lower Band'], color='#783eff', linestyle='-') ## Snap to Cursor ##
print('Variability is %2.2f.' % (variability)) # Dynamic variables # ##################################################### outputFilename = 'Scenario' + datetime.datetime.now().strftime('%y%V.%d%H%M') outputFilepath = plotsPath + outputFilename graphsCreated = [] stocks = [] time = range(1, args.days + 1) # Create stock data for all stocks for i in range(args.stocks): data = [] price = random.choice(range(1, args.maxPrice + 1)) for index in time: data.append(price) price = GetRandomPrice(price, 1, args.maxPrice, variability) stocks.append(numpy.around(data)) print(gameStocks[i]) print(data) # PLOTS # ##################################################### fig = plt.figure(figsize=(16.0, 9.0)) plot1 = plt.subplot() # Plot all stocks for index in range(args.stocks): plt.plot(time, stocks[index]) plt.plot(time, stocks[index], gameStocksTickers[index],
def getNews(df,s1,e1,q1): try: newsapi = NewsApiClient(api_key='Your Key') except: newsapi = NewsApiClient(api_key='Your another key in case of exception') try: news = newsapi.get_everything(q=q1,from_param=s1, to=e1,language='en',sort_by='popularity',page_size=100,page=1) #arranging the news articles in a numpy array news_data = [] #np.array(['publishedAt','title','description','content','url']) index = 0 for i in news['articles']: k=0 data = [] while k!=1: data.append(i["publishedAt"]) data.append(i["title"]) data.append(i["description"]) data.append(i["content"]) data.append(i["url"]) k=1 news_data.append(data) nd = np.array(news_data) data = pd.DataFrame(nd,columns=['Date','Title','Description','Content','URL']) #Spliting the date and time of each data field. new = data["Date"].str.split("T", n = 1, expand = True) new1 = new[1].str.split("Z", n=1,expand=True) data = data.drop("Date",axis=1) data.insert(loc=0, column='Date', value=new[0]) data.insert(loc=1, column='Time', value=new1[0]) data['Date'] =pd.to_datetime(data.Date) data = data.sort_values(by='Date') dff=data[['Date','Title']].groupby('Date').sum() lTemp={'Date':[],'Title':[]} for i in range(len(df)): lTemp['Date'].append(df.index[i]) lTemp['Title'].append(" ") for i in range(len(lTemp['Date'])): if lTemp['Date'][i] in list(dff.index): lTemp['Title'][i]=dff['Title'][list(dff.index).index(lTemp['Date'][i])] else: pass lTemp=pd.DataFrame(lTemp) print(lTemp.head()) freq_Vec=createVector(lTemp) if freq_Vec.shape[1]>657: freq_Vec=freq_Vec.tocsr()[:,0:658] else: q=np.array(freq_Vec.todense()) b=np.zeros((freq_Vec.shape[0],657-freq_Vec.shape[1])) p = np.concatenate((q,b),axis=1) freq_Vec = csr_matrix(p) X_test = freq_Vec.toarray() mean = np.mean(X_test) X_test -= mean prediction=predict_news(X_test) ypred=prediction pol_score = [] maxScore=max(list(df['Volume'])) minScore=min(list(df['Volume'])) finScore=(maxScore+minScore)//2 #prediction=predict_news(X_test) #ypred=prediction pol_score = [] for i in range(len(ypred)): pol_score.append((ypred[i,1]-ypred[i,0])*finScore) df['Polarity']=pol_score return df except Exception as e: print(e) df['Polarity']=[0 for i in range(len(df))] try: newsapi = NewsApiClient(api_key='Your another Key') news = newsapi.get_everything(q=q1,from_param=s1, to=e1,language='en',sort_by='popularity',page_size=100,page=1) print(news,"1") #arranging the news articles in a numpy array news_data = [] #np.array(['publishedAt','title','description','content','url']) index = 0 for i in news['articles']: k=0 data = [] while k!=1: data.append(i["publishedAt"]) data.append(i["title"]) data.append(i["description"]) data.append(i["content"]) data.append(i["url"]) data.append(i["urlToImage"]) k=1 news_data.append(data) nd = np.array(news_data) data = pd.DataFrame(nd,columns=['Date','Title','Description','Content','URL','Image']) #Spliting the date and time of each data field. new = data["Date"].str.split("T", n = 1, expand = True) new1 = new[1].str.split("Z", n=1,expand=True) data = data.drop("Date",axis=1) data.insert(loc=0, column='Date', value=new[0]) data.insert(loc=1, column='Time', value=new1[0]) data['Date'] =pd.to_datetime(data.Date) ## data = data.sort_values(by='Date') data=data.sort_values(by=['Date']) dff=data[['Date','Title']].groupby('Date').sum() lTemp={'Date':[],'Title':[]} for i in range(len(df)): lTemp['Date'].append(df.index[i]) lTemp['Title'].append(" ") for i in range(len(lTemp['Date'])): if lTemp['Date'][i] in list(dff.index): lTemp['Title'][i]=dff['Title'][list(dff.index).index(lTemp['Date'][i])] else: pass lTemp=pd.DataFrame(lTemp) print(lTemp.head()) freq_Vec=createVector(lTemp) if freq_Vec.shape[1]>657: freq_Vec=freq_Vec.tocsr()[:,0:658] else: q=np.array(freq_Vec.todense()) b=np.zeros((freq_Vec.shape[0],657-freq_Vec.shape[1])) p = np.concatenate((q,b),axis=1) freq_Vec = csr_matrix(p) X_test = freq_Vec.toarray() mean = np.mean(X_test) X_test -= mean maxScore=max(list(df['Volume'])) minScore=min(list(df['Volume'])) finScore=(maxScore+minScore)//2 prediction=predict_news(X_test) ypred=prediction pol_score = [] for i in range(len(ypred)): pol_score.append((ypred[i,1]-ypred[i,0])*finScore) df['Polarity']=pol_score return df except: df['Polarity']=[0 for i in range(len(df))] return df return []
def getTrendingNews(q1): try: newsapi = NewsApiClient(api_key='Your Key') except: newsapi = NewsApiClient(api_key='Your another key incase of exception') end = date.today() start = date.today() - relativedelta(days=+10) s1 = str(start).split(" ")[0] e1 = str(end).split(" ")[0] try: news = newsapi.get_everything(q=q1, from_param=s1, to=e1, language='en', sort_by='popularity', page_size=100, page=1) print(news, "1") #arranging the news articles in a numpy array news_data = [ ] #np.array(['publishedAt','title','description','content','url']) index = 0 for i in news['articles']: k = 0 data = [] while k != 1: data.append(i["publishedAt"]) data.append(i["title"]) data.append(i["description"]) data.append(i["content"]) data.append(i["url"]) data.append(i["urlToImage"]) k = 1 news_data.append(data) nd = np.array(news_data) data = pd.DataFrame(nd, columns=[ 'Date', 'Title', 'Description', 'Content', 'URL', 'Image' ]) #Spliting the date and time of each data field. new = data["Date"].str.split("T", n=1, expand=True) new1 = new[1].str.split("Z", n=1, expand=True) data = data.drop("Date", axis=1) data.insert(loc=0, column='Date', value=new[0]) data.insert(loc=1, column='Time', value=new1[0]) data['Date'] = pd.to_datetime(data.Date) ## data = data.sort_values(by='Date') data = data.sort_values(by=['Date'], ascending=False) return data except Exception as e: print(e) try: newsapi = NewsApiClient(api_key='Your another Key') news = newsapi.get_everything(q=q1, from_param=s1, to=e1, language='en', sort_by='popularity', page_size=100, page=1) print(news, "1") #arranging the news articles in a numpy array news_data = [ ] #np.array(['publishedAt','title','description','content','url']) index = 0 for i in news['articles']: k = 0 data = [] while k != 1: data.append(i["publishedAt"]) data.append(i["title"]) data.append(i["description"]) data.append(i["content"]) data.append(i["url"]) data.append(i["urlToImage"]) k = 1 news_data.append(data) nd = np.array(news_data) data = pd.DataFrame(nd, columns=[ 'Date', 'Title', 'Description', 'Content', 'URL', 'Image' ]) #Spliting the date and time of each data field. new = data["Date"].str.split("T", n=1, expand=True) new1 = new[1].str.split("Z", n=1, expand=True) data = data.drop("Date", axis=1) data.insert(loc=0, column='Date', value=new[0]) data.insert(loc=1, column='Time', value=new1[0]) data['Date'] = pd.to_datetime(data.Date) ## data = data.sort_values(by='Date') data = data.sort_values(by=['Date'], ascending=False) return data except: return [] return []
def get_history(symbols, start, end, data_path, visible=False): from pandas_datareader import data as y_data """ to get Yahoo data from saved csv files. If the file does not exist for the symbol, data is read from Yahoo finance and the csv saved. symbols: symbol list start, end : datetime start/end dates data_path : datapath for csv files - use double \\ and terminate path with \\ """ symbols_ls = list(symbols) for ticker in symbols: print(ticker, ' ', end="") try: #see if csv data available data = pd.read_csv(data_path + ticker + '.csv', index_col='Date', parse_dates=True) except: #if no csv data, create an empty dataframe data = pd.DataFrame(data=None, index=[start]) #check if there is data for the start-end data range if start.toordinal() < data.index[0].toordinal() \ or end.toordinal() > data.index[-1].toordinal(): if visible: print('Refresh data.. ', ) try: new_data = y_data.get_data_yahoo(ticker, start, end) if new_data.empty == False: if data.empty == False: try: ticker_data = data.append(new_data).groupby( level=0, by=['rownum']).last() except: print('Merge failed.. ') else: ticker_data = new_data try: ticker_data.to_csv(data_path + ticker + '.csv') if visible: print(' UPDATED.. ') except: print('Save failed.. ') else: if visible: print('No new data.. ') except: print('Download failed.. ') # remove symbol from list symbols_ls.remove(ticker) else: if visible: print('OK.. ') pass pdata = pd.Panel(dict((symbols_ls[i], pd.read_csv(data_path + symbols_ls[i] + '.csv',\ index_col='Date', parse_dates=True).sort_index(ascending=True)) for i in range(len(symbols_ls))) ) return pdata.ix[:, start:end, :]