import traceback import time from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from CommonServices.EmailService import EmailSender from CommonServices.RetryDecor import retry from CommonServices.LogCreater import CreateLogger from CommonServices.WebdriverServices import masterclass logger = CreateLogger().createLogFile(dirName='HoldingsScraperLogs/', logFileName="-HoldingsDataLogs.log", loggerName='TickerListLogger') class Download523TickersList(): def __init__(self): self.driver = None @retry(Exception, total_tries=2, initial_wait=2, backoff_factor=2) def webdriver_login_etfdb(self): driverclass = masterclass() driverclass.initialisewebdriver() driverclass.logintoetfdb() self.driver = driverclass.driver @retry(Exception, total_tries=2, initial_wait=2, backoff_factor=2) def open_webpage_for_list(self, url): try: self.driver.get(url)
import datetime import sys, traceback sys.path.append("..") # Remove in production - KTZ import ujson import json import pandas as pd import websocket import logging import os from CommonServices.EmailService import EmailSender from CommonServices.LogCreater import CreateLogger from CommonServices import ImportExtensions logObj = CreateLogger() logger = logObj.createLogFile(dirName="ETFLiveAnalysis/",logFileName="-TradesLiveLog.log",loggerName="TradesLiveLog") try: import thread except ImportError: import _thread as thread import time from MongoDB.PerMinDataOperations import PerMinDataOperations import asyncio def on_message(ws, message): start = time.time() responses = ujson.loads(message) #dataQ = [response for response in responses if response['ev'] == 'Q'] dataAM = [response for response in responses if response['ev'] == 'AM']
import pathlib import sys # Remove in production - KTZ import traceback sys.path.append("..") # Remove in production - KTZ from CommonServices.Holidays import HolidayCheck from CommonServices.EmailService import EmailSender from CommonServices.MakeCSV import CSV_Maker import pandas as pd from datetime import datetime from datetime import timedelta from CalculateETFArbitrage.CalculateHistoricalArbitrage import ArbitrageCalculation from MongoDB.SaveArbitrageCalcs import SaveCalculatedArbitrage from MongoDB.FetchArbitrage import FetchArbitrage from CommonServices.LogCreater import CreateLogger logger = CreateLogger().createLogFile(dirName="HistoricalArbitrage/", logFileName="-ArbEventLog.log", loggerName="ArbEventLogger", filemode='a') logger2 = CreateLogger().createLogFile(dirName="HistoricalArbitrage/", logFileName="-ArbErrorLog.log", loggerName="ArbErrorLogger", filemode='a') class HistoricalArbitrageRunnerClass(): def __init__(self): self.etflist = [] self.etfwhichfailed = [] self.date = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d") self.rootpath = pathlib.Path(os.getcwd()) while str(self.rootpath).split('/')[-1] != 'ETFAnalyzer': self.rootpath = self.rootpath.parent self.csv_file_path = os.path.abspath(os.path.join(self.rootpath, 'CSVFiles/250M_WorkingETFs.csv'))
import sys, traceback sys.path.append('..') from datetime import datetime, timedelta import CommonServices.ImportExtensions from CommonServices.EmailService import EmailSender import getpass from MongoDB.MongoDBConnections import MongoDBConnectors from CommonServices.LogCreater import CreateLogger logObj = CreateLogger() logger = logObj.createLogFile(dirName='ETFLiveAnalysis/', logFileName='-DeleteScriptLog.log', loggerName='DeleteScriptLogger') sys_username = getpass.getuser() if sys_username == 'ubuntu': readWriteConnection = MongoDBConnectors( ).get_pymongo_readWrite_production_production() else: readWriteConnection = MongoDBConnectors().get_pymongo_devlocal_devlocal() db = readWriteConnection.ETF_db def delete_old_live_data_from_collection(collectionName): try: # LAST TIMESTAMP PRESENT IN THE COLLECTION if collectionName == db.TradePerMinWS: last = collectionName.find({}, { 'e': 1, '_id': 0 }).sort([('e', -1)]).limit(1) last_ts = list(last)[0]['e'] elif collectionName == db.ArbitragePerMin:
import traceback import datetime import time import pandas as pd # Custom Imports from CommonServices.EmailService import EmailSender from CommonServices.LogCreater import CreateLogger from CommonServices.MultiProcessingTasks import CPUBonundThreading from CommonServices import ImportExtensions from MongoDB.PerMinDataOperations import PerMinDataOperations from MongoDB.Schemas import trade_per_min_WS from ETFLiveAnalysisProdWS.Helper.CalculationHelper import LiveHelper, tradestruct from functools import partial logObj = CreateLogger() logger = logObj.createLogFile(dirName="ETFLiveAnalysis/", logFileName="-CalculateArbPerMinLog.log", loggerName="CalculateArbPerMinLog") class ArbPerMin(): def __init__(self, etflist, etfdict, tickerlist): self.etflist = etflist # Only used once per day self.etfdict = etfdict # Only used once per day self.tickerlist = tickerlist self.trade_dict = { } # Maintains only 1 copy throughout the day and stores {Ticker : trade objects} self.TradesDataDfPreviousMin = None self.TradesDataDfCurrentMin = None self.helperobj = LiveHelper()
import os import pathlib import getpass import traceback import pandas as pd from datetime import datetime from mongoengine.errors import NotUniqueError from CommonServices.LogCreater import CreateLogger from CommonServices.EmailService import EmailSender from MongoDB.Schemas import etfholdings_collection logger = CreateLogger().createLogFile(dirName='HoldingsScraperLogs/', logFileName="-HoldingsDataLogs.log", loggerName='DataCleanFeed') class PullandCleanData: def __init__(self): self.rootpath = pathlib.Path(os.getcwd()) while str(self.rootpath).split('/')[-1] != 'ETFAnalyzer': self.rootpath = self.rootpath.parent self.savingpath = os.path.abspath(os.path.join(self.rootpath, 'ETFDailyData' + '/' + datetime.now().strftime("%Y%m%d"))) self.system_username = getpass.getuser() self.coll = etfholdings_collection def readfilesandclean(self, etfname, etfdescdf): try: # Trying Dict Comprehension for file checking and loading to check etf file in directory x = {f.split('-')[0]: f for f in os.listdir(self.savingpath)} if etfname in x.keys():
import sys, traceback sys.path.append('../..') from datetime import datetime, timedelta import CommonServices.ImportExtensions from CommonServices.EmailService import EmailSender import getpass from MongoDB.MongoDBConnections import MongoDBConnectors from CommonServices.LogCreater import CreateLogger logObj = CreateLogger() logger = logObj.createLogFile(dirName='HistoricalArbitrage/', logFileName='-HistoricDeleteScriptLog.log', loggerName='HistoricDeleteScriptLogger') sys_username = getpass.getuser() if sys_username == 'ubuntu': readWriteConnection = MongoDBConnectors( ).get_pymongo_readWrite_production_production() else: readWriteConnection = MongoDBConnectors().get_pymongo_devlocal_devlocal() db = readWriteConnection.ETF_db def delete_old_live_data_from_collection(collectionName): try: # LAST TIMESTAMP PRESENT IN THE COLLECTION last = collectionName.find({}, { 'dateForData': 1, '_id': 0 }).sort([('dateForData', -1)]).limit(1) last_date = list(last)[0]['dateForData']
import sys, traceback sys.path.append("..") import datetime import pandas as pd from CommonServices.MultiProcessingTasks import CPUBonundThreading from CommonServices.ThreadingRequests import IOBoundThreading from CommonServices.LogCreater import CreateLogger logger = CreateLogger().createLogFile(dirName='ETFLiveAnalysis/', logFileName='-QuotesLiveFetchLog.log', loggerName='QuotesLiveFetch') from PolygonTickData.PolygonCreateURLS import PolgonDataCreateURLS from MongoDB.PerMinDataOperations import PerMinDataOperations class QuotesLiveFetcher(): def __init__(self): # self.etflist = pd.read_csv('../CSVFiles/250M_WorkingETFs.csv').columns.to_list() self.etflist = [ 'SPY', 'VOO', 'QQQ', 'IVV', 'IJR', 'VO', 'VGT', 'XLK', 'XLF', 'SCHX' ] self.getUrls = [ PolgonDataCreateURLS().PolygonLastQuotes(etf) for etf in self.etflist ] def getDataFromPolygon(self, methodToBeCalled=None, getUrls=None): # Calling IO Bound Threading to fetch data for URLS if methodToBeCalled == None or getUrls == None: logger.debug('Either methodToBeCalled or getUrls not supplied')
from selenium.common.exceptions import NoSuchElementException from urllib3.exceptions import NewConnectionError, MaxRetryError from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait from CommonServices.LogCreater import CreateLogger from CommonServices.EmailService import EmailSender from CommonServices.RetryDecor import retry from CommonServices.WebdriverServices import masterclass from MongoDB.MongoDBConnections import MongoDBConnectors logger = CreateLogger().createLogFile(dirName='HoldingsScraperLogs/', logFileName="-HoldingsDataLogs.log", loggerName='DownloadHoldingsLogger') class PullHoldingsListClass(object): def __init__(self): self.system_username = getpass.getuser() if self.system_username == 'ubuntu': ''' Production to Production readWrite ''' self.conn = MongoDBConnectors( ).get_pymongo_readWrite_production_production() else: ''' Dev Local to Production Read Only ''' # self.conn = MongoDBConnectors().get_pymongo_readonly_devlocal_production() ''' Dev Local to Dev Local readWrite ''' self.conn = MongoDBConnectors().get_pymongo_devlocal_devlocal()
import socket import sys sys.path.append("../") import time import traceback from functools import partial from itertools import chain from CommonServices.MultiProcessingTasks import multi_processing_method, CPUBonundThreading from PerSecLive.Helpers import * from MongoDB.MongoDBConnections import MongoDBConnectors from pymongo import ASCENDING, DESCENDING from CommonServices.LogCreater import CreateLogger logObj = CreateLogger() logger = logObj.createLogFile(dirName="PerSecLive/", logFileName="-PerSecLiveCalcLog.log", loggerName="PerSecLiveCalcLog") sys_private_ip = socket.gethostbyname(socket.gethostname()) if sys_private_ip == '172.31.76.32': connection = MongoDBConnectors( ).get_pymongo_readWrite_production_production() else: connection = MongoDBConnectors().get_pymongo_devlocal_devlocal() collection = connection.ETF_db.PerSecLiveArbitrage collection.create_index([("ETFName", ASCENDING), ("End_Time", DESCENDING)]) def main_runner(etf_list, _date, ticker_list=None):
import datetime import json import pandas as pd import schedule import time # Custom Imports # from CommonServices.EmailService import EmailSender from ETFLiveAnalysisProdWS.TickListsGenerator import ListsCreator from ETFLiveAnalysisProdWS.CalculatePerMinArb import ArbPerMin from CommonServices.LogCreater import CreateLogger from CommonServices import ImportExtensions from MongoDB.PerMinDataOperations import PerMinDataOperations from MongoDB.SaveArbitrageCalcs import SaveCalculatedArbitrage logObj = CreateLogger() logger = logObj.createLogFile(dirName="ETFLiveAnalysis/", logFileName="-PerMinCaller.log", loggerName="PerMinCallerLogs") class PerMinAnalysis(): def __init__(self): self.perMinDataObj = PerMinDataOperations() self.spreadDF = pd.DataFrame() def get_ts_for_fetching_data(self): ####################################################### # UTC Timestamps for pulling data from QuotesLiveData DB, below: ####################################################### end_dt = datetime.datetime.now().replace(second=0, microsecond=0)
import logging from flask import has_request_context, request from logging.handlers import RotatingFileHandler from CommonServices.LogCreater import CreateLogger class RequestFormatter(logging.Formatter): def format(self, record): if has_request_context(): record.url = request.url record.remote_addr = request.remote_addr else: record.url = None record.remote_addr = None return super().format(record) def return_formatter(): return RequestFormatter( '[%(asctime)s] %(levelname)s %(remote_addr)s requested %(url)s in %(module)s: %(message)s' ) custom_server_logger = CreateLogger().createLogFile( dirName="Server/", logFileName="-ServerLog.log", loggerName="ServerLogger", filemode='a', user_format=return_formatter())
import sys import json import datetime import traceback sys.path.append("..") from CommonServices.MakeCSV import CSV_Maker import pandas as pd from CalculateETFArbitrage.Helpers.LoadEtfHoldings import LoadHoldingsdata from CommonServices.MultiProcessingTasks import CPUBonundThreading from MongoDB.Schemas import etfholdings_collection from CommonServices.LogCreater import CreateLogger logObj = CreateLogger() logger = logObj.createLogFile(dirName="ETFLiveAnalysis/", logFileName="-TickListGenerator.log", loggerName="TickListGenerator") class ListsCreator(): def convertDataToDict(self, df, etfname): df.drop(columns=['TickerName'], inplace=True) df.rename(columns={ 'TickerSymbol': 'symbol', 'TickerWeight': 'weight' }, inplace=True) res = {} res['HoldingsList'] = df['symbol'].to_list() res['ETFHoldingsData'] = {etfname: [df.to_dict()]} return res
sys.path.append("..") # Remove in production - KTZ import json import asyncio from aiohttp import ClientSession import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry from MongoDB.SaveFetchQuotesData import MongoTradesQuotesData from PolygonTickData.Helper import Helper from CommonServices.LogCreater import CreateLogger logger = CreateLogger().createLogFile(dirName='HistoricalArbitrage/', logFileName='-ArbEventLog.log', loggerName='TradesQuotesEventLogger') class FetchPolygonData(object): """Fetch and Store Methods for Trades/Quotes Data from Polygon.io""" def __init__(self, date=None, end_time='21:00:00', end_time_loop='20:00:00', polygon_method=None, symbol_status=None, collection_name=None, insert_into_collection=None): self.helperObj = Helper() self.date = date
from time import perf_counter from datetime import datetime import pandas as pd import os # Use absolute import paths from CommonServices.LogCreater import CreateLogger from HoldingsDataScripts.Download523TickersList import Download523TickersList from HoldingsDataScripts.DownloadHoldings import DownloadsEtfHoldingsData from HoldingsDataScripts.DataCleanFeed import PullandCleanData from CommonServices.EmailService import EmailSender from CommonServices.DirectoryRemover import Directory_Remover from CommonServices.MultiProcessingTasks import multi_processing_method t1_start = perf_counter() logger = CreateLogger().createLogFile(dirName='HoldingsScraperLogs/', logFileName="-HoldingsDataLogs.log", loggerName='HoldingsLogger') class holdingsProcess(): def __init__(self): self.rootpath = pathlib.Path(os.getcwd()) while str(self.rootpath).split('/')[-1] != 'ETFAnalyzer': self.rootpath = self.rootpath.parent self.csv_file_path = os.path.abspath( os.path.join(self.rootpath, 'CSVFiles/250M_WorkingETFs.csv')) self.ticker_description_path = os.path.abspath( os.path.join( self.rootpath, 'ETFDailyData/ETFTickersDescription/' + datetime.now().strftime("%Y%m%d") +
import sys, traceback sys.path.append('../..') from CommonServices.ImportExtensions import * import pandas as pd import getpass, datetime from pymongo import * from MongoDB.MongoDBConnections import MongoDBConnectors from FlaskAPI.Components.ETFArbitrage.ETFArbitrageMain import AnalyzeArbitrageDataForETF from CommonServices.LogCreater import CreateLogger logger = CreateLogger().createLogFile(dirName='HistoricalArbitrage/', logFileName='-PNLLog.log', loggerName='PNLLogger') class CalculateAndSavePnLData(): def __init__(self): self.sysUserName = getpass.getuser() if self.sysUserName == 'ubuntu': self.connforthis = MongoDBConnectors( ).get_pymongo_readWrite_production_production() else: self.connforthis = MongoDBConnectors( ).get_pymongo_readonly_devlocal_production() self.connforthis = MongoDBConnectors().get_pymongo_devlocal_devlocal() self.arbitragecollection = self.connforthis.ETF_db.ArbitrageCollectionNew ######################################################################################## # Use to populate DB for the first time, for all dates which are not present ########################################################################################