def yeda8(fileloc, filetype, ticker): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "/space/common/workspace/phase2_output/program8_out/matplotlib/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") df1 = pf.get_tickerData(fileloc, filetype, ticker, "Q") df2 = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got data. Generating plot...") ################# # Plotting time # ################# if (df1.empty): print("At date " + date + ", ticker" + ticker + " had no quote data\n") if (df2.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") return fig, ax = plt.subplots(figsize=(20, 8)) ax.plot(df1.Time, df1['Ask Size Dec'], color="blue", linewidth=1.0, linestyle='-', label='Ask Volume') ax.plot(df1.Time, df1['Bid Size Dec'], color="red", linewidth=1.0, linestyle='-', label='Bid Volume') ax.set_xlabel('Time (UTC)') ax.set_ylabel('Volume') title_date = df1.Time[len(df1) - 1].strftime('%b %-d, %Y') ax.set_title(ticker + " | " + "Volume | " + title_date) ax.spines['right'].set_color('none') ax.spines['top'].set_color('none') #setting major locator alldays = mdates.HourLocator(interval=1) # 3H interval ax.xaxis.set_major_locator(alldays) ax.xaxis.set_major_formatter(mdates.DateFormatter('%I %p')) #setting minor locator hoursLoc = mdates.HourLocator(interval=30) ax.xaxis.set_minor_locator(hoursLoc) ax.xaxis.set_minor_formatter(mdates.DateFormatter('%M')) ax.legend() imgname = ticker + "_" + "quotes_" + "Volume" + filetype + "_" + date + ".png" fig.savefig(dirpath + imgname) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
def program82(fileloc, filetype, ticker): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "/space/common/workspace/phase2_output/program8_out/plotly/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") data1 = pf.get_tickerData(fileloc, filetype, ticker, "Q") data2 = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got data. Generating plot...") ################# # Plotting time # ################# if (data2.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") return # Plotting the data trace = go.Scatter(x=data2.Time, y=data2["Trade Price"], line={'color': 'blue'}, name="Trade Price") title_date = data2.Time[len(data2) - 1].strftime('%b %-d, %Y') layout = go.Layout(xaxis={ 'title': 'Time (UTC)', 'type': 'date', 'tickformat': '%I:%M:%S %p' }, yaxis={'title': 'Trade Price'}, title=ticker + " | Trade | " + title_date) fig = go.Figure(data=[trace], layout=layout) # Output the plot imgname = ticker + "_" + "quotes_" + "Trade Price_" + filetype + "_" + date py.plot(fig, filename=dirpath + imgname + ".html", image="png", image_filename=imgname, image_width=1024, image_height=768, auto_open=False, show_link=False) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
def program6(fileloc1, fileloc2, filetype1, filetype2, ticker): init_tm = time() # Make dir fsplit = fileloc1.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "phase1_output/program6_out/plotly/" + filetype1 + "/" + filetype2 + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") data1 = pf.get_tickerData(fileloc1, filetype1, "X:SXBTUSD", "T") print(ticker) data2 = pf.get_tickerData(fileloc2, filetype2, ticker, "T") print("Got data. Generating plot...") ################# # Plotting time # ################# if (data1.empty or data2.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") return trace = go.Scatter(x=data1.Time, y=data2["Trade Price"] - data1["Trade Price"]) title_date = data1.Time[len(data1) - 1].strftime('%b %-d, %Y') layout = go.Layout(xaxis={ 'title': 'Time (UTC)', 'type': 'date', 'tickformat': '%I:%M:%S %p' }, yaxis={'title': 'Spread'}, title=ticker + " | " + "Spread | " + title_date) fig = go.Figure(data=[trace], layout=layout) # Output the plot imgname = ticker + "_" + "spread_" + filetype1 + "_" + filetype2 + "_" + date py.plot(fig, filename=dirpath + imgname + ".html", image="png", image_filename=imgname, image_width=1024, image_height=768, auto_open=False, show_link=False) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
def ppppp6(fileloc, filetype, ticker): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "phase1_output/program6_out/plotly/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) #print(ticker) data = pf.get_tickerData(fileloc, filetype, ticker, "T") #print("Got data. Generating plot...") ################# # Plotting time # ################# if (data.empty): print(date + ", ticker " + ticker + " no trade data\n", file=txt) #data.write(date + ", ticker " + ticker + " no trade data\n") return else: print(date + ", ticker " + ticker + " no trade data\n", file=txt) #data.write(date + ", ticker " + ticker + " had trade data\n") return
def yeda2(fileloc, filetype): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] dirpath = "program2_mpl_out/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) # Get List of tickers pf.get_tickerList(date, filetype) for ticker in projLists.ticker_list: data_starttm = time() data = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got", ticker, "data DF in", time() - data_starttm, "Seconds") ################# # Plotting time # ################# if (data.empty): message = "At date " + date + ", ticker" + ticker + " had no trade data\n" print(message) with open("program2_mpl_" + filetype + "_log.txt", 'a+') as f: f.write(message) continue fig, ax = plt.subplots(figsize=(20, 8)) ax.plot(data.Time, data["Trade Price"], color="red", linewidth=1.0, linestyle='-') ax.set_xlabel('Time (UTC)') ax.set_ylabel('Price') title_date = data.Time[len(data) - 1].strftime('%b %-d, %Y') ax.set_title(ticker + " | " + "Trades | " + title_date) ax.spines['right'].set_color('none') ax.spines['top'].set_color('none') #setting major locator alldays = mdates.HourLocator(interval=1) # 3H interval ax.xaxis.set_major_locator(alldays) ax.xaxis.set_major_formatter(mdates.DateFormatter('%I %p')) #setting minor locator hoursLoc = mdates.HourLocator(interval=30) ax.xaxis.set_minor_locator(hoursLoc) ax.xaxis.set_minor_formatter(mdates.DateFormatter('%M')) ax.legend() imgname = ticker + "_" + "trades_" + filetype + "_" + date + ".png" fig.savefig(dirpath + imgname) plt.close() print("Ticker", ticker, "at date", date, "done") print(date, "plots finished in", time() - init_tm, "seconds")
def program5a(fileloc, filetype, ticker, quotetype): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] dirpath = "output/program5a_out/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") data = pf.get_tickerData(fileloc, filetype, ticker, "Q") print("Got data. Generating plot...") ################# # Plotting time # ################# if (data.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") return # Make a trace for each venue traces = [] for venue in data["Contributor Id"].unique(): trace = go.Scatter(x=data.Time[data["Contributor Id"] == venue], y=data[quotetype + " Price"][data["Contributor Id"] == venue], name=venue) traces.append(trace) title_date = data.Time[len(data) - 1].strftime('%b %-d, %Y') layout = go.Layout(xaxis={ 'title': 'Time (UTC)', 'type': 'date', 'tickformat': '%I:%M:%S %p' }, yaxis={'title': 'Trade Price'}, title=ticker + " | " + quotetype + " | " + title_date, showlegend=True) fig = go.Figure(data=traces, layout=layout) # Output the plot imgname = ticker + "_" + "quotes_" + quotetype + "_" + filetype \ + "_" + date py.plot(fig, filename=dirpath + imgname + ".html", image="png", image_filename=imgname, image_width=1024, image_height=768, auto_open=False, show_link=False) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
def program3(fileloc, filetype): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] dirpath = "program3_out/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) # Get List of tickers pf.get_tickerList(date, filetype) for ticker in projLists.ticker_list: data_starttm = time() data = pf.get_tickerData(fileloc, filetype, ticker, "Q") print("Got", ticker, "data DF in", time() - data_starttm, "Seconds") ################# # Plotting time # ################# if(data.empty): message = "At date " + date + ", ticker" + ticker + " had no quote data\n" print(message) with open("program3_" + filetype + "_log.txt", 'a+') as f: f.write(message) continue trace1 = go.Scatter(x = data.Time, y = data["Bid Price"], line = {'color': 'red'}, name = "Bid Price") trace2 = go.Scatter(x = data.Time, y = data["Ask Price"], line = {'color': 'blue'}, name = "Ask Price") title_date = data.Time[len(data) - 1].strftime('%b %-d, %Y') layout = go.Layout(xaxis={'title': 'Time (UTC)', 'type': 'date', 'tickformat': '%I:%M:%S %p'}, yaxis={'title': 'Price'}, title=ticker + " | Quotes | " + title_date) fig = go.Figure(data = [trace1, trace2], layout = layout) # Output the plot imgname = ticker + "_" + "quotes_" + filetype + "_" + date py.plot(fig, filename=dirpath + imgname + ".html", image="png", image_filename=imgname, image_width=1024, image_height=768, auto_open=False, show_link=False) print("Ticker", ticker, "at date", date, "done") print(date, "plots finished in", time() - init_tm, "seconds")
def call(fileloc, filetype, ticker): # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "phase3_output/program3.2_out/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") df = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got data. Generating plot...") return(df)
def yeda2(fileloc, filetype, ticker): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] dirpath = "output/program2_out/matplotlib/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") df = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got data. Generating plot...") ################# # Plotting time # ################# if(df.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") return fig, ax = plt.subplots(figsize=(20,8)) ax.plot(df.Time, df["Trade Price"], color = "red", linewidth = 1.0, linestyle = '-') ax.set_xlabel('Time (UTC)') ax.set_ylabel('Price') title_date = df.Time[len(df) - 1].strftime('%b %-d, %Y') ax.set_title(ticker + " | " + "Trades | " + title_date) ax.spines['right'].set_color('none') ax.spines['top'].set_color('none') #setting major locator alldays = mdates.HourLocator(interval = 1) # 3H interval ax.xaxis.set_major_locator(alldays) ax.xaxis.set_major_formatter(mdates.DateFormatter('%I %p')) #setting minor locator hoursLoc = mdates.HourLocator(interval=30) ax.xaxis.set_minor_locator(hoursLoc) ax.xaxis.set_minor_formatter(mdates.DateFormatter('%M')) ax.legend() imgname = ticker + "_" + "trades_" + filetype + "_" + date + ".png" fig.savefig(dirpath + imgname) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
def zzg(fileloc, filetype, ticker, quotetype): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "/space/common/workspace/phase1_output/program5a_out/" + filetype + "/" + date + "_" + quotetype + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") df = pf.get_tickerData(fileloc, filetype, ticker, "Q") print("Got data. Generating plot...") df["Time"] = pd.to_datetime(df["Time"]) df = df.set_index("Time") dfgroupby = df[[quotetype + " Price", "Contributor Id"]].groupby("Contributor Id") fig, ax = plt.subplots(figsize=(20, 8)) dfgroupby[quotetype + " Price"].plot(ax=ax, legend=True) ax.set_xlabel('Time (UTC)') ax.set_ylabel(quotetype + " Price") title_date = df.index[len(df) - 1].strftime('%b %-d, %Y') ax.set_title(ticker + " | " + quotetype + " | " + title_date) ax.spines['right'].set_color('none') ax.spines['top'].set_color('none') #setting major locator alldays = mdates.HourLocator(interval=1) # 3H interval ax.xaxis.set_major_locator(alldays) ax.xaxis.set_major_formatter(mdates.DateFormatter('%I %p')) #setting minor locator hoursLoc = mdates.HourLocator(interval=30) ax.xaxis.set_minor_locator(hoursLoc) ax.xaxis.set_minor_formatter(mdates.DateFormatter('%M')) imgname = ticker + "_" + quotetype + '_' + filetype + "_" + date + ".png" fig.savefig(dirpath + imgname) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
def get_ohlc(fileloc, filetype, ticker): """ Get open-high-low-close of ticker for one day Args: Returns: """ fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] formatted_date = '-'.join([date[:4], date[4:6], date[6:]]) print("Getting", ticker, "data from date", date) with open(fileloc, 'r') as finput: df = pf.get_tickerData(finput, filetype, ticker, "T") o = df["Trade Price"][0] h = df["Trade Price"].max() l = df["Trade Price"].min() c = df["Trade Price"][df.shape[0] - 1] print("Got ohlc at date", date) return [formatted_date, o, h, l, c]
def program5a(fileloc, filetype, quotetype): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] dirpath = "program5a_out/" + filetype + "/" + date + "/" + quotetype + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) # Get List of tickers pf.get_tickerList(date, filetype) for ticker in projLists.ticker_list: data_starttm = time() data = pf.get_tickerData(fileloc, filetype, ticker, "Q") print("Got", ticker, "data DF in", time() - data_starttm, "Seconds") ################# # Plotting time # ################# if (data.empty): message = "There is no " + quotetype + " data for " + ticker + " at " + date + "\n" print(message) with open("program5a_" + filetype + "_log.txt", 'a+') as f: f.write(message) continue title_date = data.Time[len(data) - 1].strftime('%b %-d, %Y') layout = go.Layout(xaxis={ 'title': 'Time (UTC)', 'type': 'date', 'tickformat': '%I:%M:%S %p' }, yaxis={'title': 'Price'}, title=ticker + " | " + quotetype + " | " + title_date, showlegend=True) # Make a trace for each venue traces = [] for venue in data["Contributor Id"].unique(): trace = go.Scatter( x=data.Time[data["Contributor Id"] == venue], y=data[quotetype + " Price"][data["Contributor Id"] == venue], name=venue) traces.append(trace) # Output html file fig = go.Figure(data=traces, layout=layout) imgname = ticker + "_" + "quotes_" + quotetype + "_" + filetype \ + "_" + date py.plot(fig, filename=dirpath + imgname + ".html", image="png", image_filename=imgname, image_width=1024, image_height=768, auto_open=False, show_link=False) print("Ticker", ticker, "at date", date, "done") print(date, "plots finished in", time() - init_tm, "seconds")
option2 = "T" else: option2 = "Q" else: print("Invalid Input") option = option1 + " Price" if (filetype == "A"): fileloc = "/space/data/new/PLUSTICK_1619_" + date + ".txt" if (filetype == "B"): fileloc = "/space/data/new/PLUSTICK_FI_1356_" + date + ".txt" if (filetype == "C"): fileloc = "/space/data/new/PLUSTICK_FUTURES_666_" + date + ".txt" if (filetype == "D"): fileloc = "/space/data/new/PLUSTICK_FUTURES_680_" + date + ".txt" df = pf.get_tickerData(fileloc, filetype, ticker, option2) except IndexError: print("Type 'list' to get a list of valid inputs") filetype = pf.get_validInput("Type A or B or C or D files: ", 4) date = pf.get_validInput("Enter Date in yyyymmdd: ", 0, filetype=filetype) ticker = pf.get_validInput("Enter One Ticker: ", 1) ticker = ticker[0] while True: option1 = input("Enter Trade, Bid, or Ask: ") if (option1 in ["Trade", "Bid", "Ask"]): if (option1 == "Trade"): option2 = "T" else:
def yeda34(fileloc, filetype, date, option1, option2, tickerlist): init_tm = time() val_ticker = [] for ticker in tickerlist: df = pf.get_tickerData(fileloc, filetype, ticker, option2) if (df.empty): print('empty') continue try: if option1 == "Trade": vol = df['Trade Size Dec'] if option1 == "Ask": vol = df['Ask Size Dec'] if option1 == "Bid": vol = df['Bid Size Dec'] except KeyError: continue val_ticker.append(ticker) if not val_ticker: print("At date " + date + ", instrument " + ins + " had no " + option1 + " volume size data\n") raise SystemExit print("Getting data...") while True: print("Valid ticker:", val_ticker) ticker = input("Enter ticker: ") if ticker in val_ticker: break else: print("Invalid Input") ################# # Plotting time # ################# # Directory stuff os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "/".join([ "/space/common/workspace/phase3_output/program4_out", filetype, date, ticker ]) + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Generatting plot...") fig, ax = plt.subplots(figsize=(20, 8)) ax.set_xlabel('Time (UTC)') ax.set_ylabel('Volume') df = pf.get_tickerData(fileloc, filetype, ticker, option2) if option1 == "Trade": vol = df['Trade Size Dec'] if option1 == "Ask": vol = df['Ask Size Dec'] if option1 == "Bid": vol = df['Bid Size Dec'] df["Time"] = pd.to_datetime(df["Time"]) ax.plot(df['Time'], vol, linewidth=0.5, linestyle='-', label=ticker) title_date = df.Time[len(df) - 1].strftime('%b %-d, %Y') ax.set_title(ticker + " | " + option1 + " Volume | " + title_date) ax.spines['right'].set_color('none') ax.spines['top'].set_color('none') #setting major locator alldays = mdates.HourLocator(interval=1) # 3H interval ax.xaxis.set_major_locator(alldays) ax.xaxis.set_major_formatter(mdates.DateFormatter('%I %p')) #setting minor locator hoursLoc = mdates.HourLocator(interval=30) ax.xaxis.set_minor_locator(hoursLoc) ax.xaxis.set_minor_formatter(mdates.DateFormatter('%M')) ax.legend() imgname = ticker + "_" + option1 + "_" + "Volume" + filetype + "_" + date + ".png" fig.savefig(dirpath + imgname) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")
if (filetype == "A"): fileloc = "/space/data/new/PLUSTICK_1619_" + date + ".txt" CI = "Contributor Id" if (filetype == "B"): fileloc = "/space/data/new/PLUSTICK_FI_1356_" + date + ".txt" CI = "Contributor Id" if (filetype == "C"): fileloc = "/space/data/new/PLUSTICK_FUTURES_666_" + date + ".txt" CI = "Part Code" if (filetype == "D"): fileloc = "/space/data/new/PLUSTICK_FUTURES_680_" + date + ".txt" CI = "Part Code" print("Getting data...") vol = pf.get_tickerData(fileloc, filetype, ticker, "Q") df = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got data. Generating plot...") if(vol.empty): print("At date " + date + ", ticker" + ticker + " had no quote data\n") if(df.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") df[CI] = df[CI].fillna('unknown') venuelist = df[CI].unique() yeda10(filetype, df)
def program5b(fileloc, filetype, ticker): init_tm = time() # Make dir fsplit = fileloc.split('_') date = fsplit[len(fsplit) - 1][:-4] os.chdir(str(pathlib.Path.home()) + "/workspace/") dirpath = "/space/common/workspace/phase1_output/program5b_out/" + filetype + "/" + date + "/" pathlib.Path(dirpath).mkdir(parents=True, exist_ok=True) print("Getting data...") data = pf.get_tickerData(fileloc, filetype, ticker, "T") print("Got data. Generating plot...") ################# # Plotting time # ################# if(data.empty): print("At date " + date + ", ticker" + ticker + " had no trade data\n") return # Make a trace for each venue if (filetype == "A"): CI = "Contributor Id" if (filetype == "B"): CI = "Contributor Id" if (filetype == "C"): CI = "Part Code" if (filetype == "D"): CI = "Part Code" traces = [] data[CI] = data[CI].fillna('un') # print(data[CI]) for venue in data[CI].unique(): trace = go.Scatter(x = data.Time[data[CI] == venue], y = data["Trade Price"][data[CI] == venue], name = venue) traces.append(trace) title_date = data.Time[len(data) - 1].strftime('%b %-d, %Y') layout = go.Layout(xaxis={'title': 'Time (UTC)', 'type': 'date', 'tickformat': '%I:%M:%S %p'}, yaxis={'title': 'Trade Price'}, title=ticker + " | " + "Trades | " + title_date) fig = go.Figure(data = traces, layout = layout) # Output the plot imgname = ticker + "_" + "tradesV2_" + filetype + "_" + date py.plot(fig, filename=dirpath + imgname + ".html", image="png", image_filename=imgname, image_width=1024, image_height=768, auto_open=False, show_link=False) print("Ticker", ticker, "at date", date, "finished in", time() - init_tm, "seconds")