def __init__(self):
#TODO move parse pid here later
self.mtd = marketdata.MarketData()
self.cfg = ms_config.MsDataCfg("")
self.symbolLstFile = self.cfg.getDataConfig("marketdata")
self.initParams()
pass
def main():
strike = 40
expiry = 1.
spot = 41
rate = 0.08
volatility = 0.3
dividend = 0.0
steps = 365
paths = 10000
payoff = op.put_payoff
pricer = lsp.LS_Pricer
mcarlo = mc.mcpath
regression = reg.regress
option = op.VanillaOption(expiry, strike, payoff)
engine = engines.LSPricingEngine(steps, paths, pricer, mcarlo, regression)
data = md.MarketData(rate, spot, volatility, dividend)
testoption = op.OptionFacade(option, engine, data)
price = testoption.price()
print("The price of the option is {:.3f}.".format(price))
def __init__(self):
pandas.set_option('display.max_columns', 50)
pandas.set_option('display.precision', 3)
pandas.set_option('display.expand_frame_repr', False) #expand wide dataframe
pandas.set_option('display.max_rows', 1500)
#pandas.set_option('display.width', 100)
#pandas.set_option('max_colwidth', 600)
#pandas.set_option('display.float_format', lambda x: '%.3f' % x)
self.outputpath = "../result/"
#self.mscfg = "./marketscan.cfg" #??
self.mfeed = ms_feed.ms_feed()
self.backtest = ms_backtest.ms_backtest()
self.csvchart = ms_csvchart.ms_csvchart()
self.mtd = marketdata.MarketData()
self.datacfg = ms_config.MsDataCfg("")
self.cachepath = self.datacfg.getDataConfig("folder","../cache/")
self.sgyInxDct={}
self.rawData = {} # MarketScan.RawData()
self.app_param = ms_paramparser.AppParam()
self.param_parser = ms_paramparser.ms_paramparser()
global g_marketscan
g_marketscan = self
# cahce the last result
return
def __init__(self):
#self.rankPattern = '[\d\D]*Zacks.*Rank[/s]*: (.*) <sup class=[\d\D]*'
#self.rankPattern = '[\d\D]*Zacks.*Rank[/s]*: (.*)[\n]*<sup class=[\d\D]*'
#<p>Zacks Rank : 2-Buy <sup class=" AAPL
#self.rankPattern = '[\d\D]*Zacks[\D]*Rank[/s]*: (.)[\d\D]*'
self.rankPattern = '[\d\D]*Zacks[\D]*Rank[\s]?: (.)[\d\D]*'
#self.erdPattern = '^window.app_data_earnings[\d\D]*\\"data\\"[ :\\[]*(.*)]' #format change
self.erdPattern = '[\d\D]*var obj = [\d\D]*earnings_announcements_earnings_table[\D]*:([\d\D]*?)] ][\d\D]*'
self.cqestCol = ['cq0', 'cq7', 'cq30', 'cq60', 'cq90']
self.columns = ['rank','indurank','indutotal','etf','abrt','abr1w','abr1m','abr2m',\
'abr3m','numbr'] + self.cqestCol
self.allcols = ['symbol', 'exg'] + self.columns
#self.pid = 0
#self.option = ""
#self.tickdf = pandas.DataFrame()
self.mtd = marketdata.MarketData()
self.cfg = ms_config.MsDataCfg("") # default = datafile, cb_config.cfg
self.mkdataFile = self.cfg.getDataConfig(
"marketdata") #"./marketdata.csv"
self.cachefolder = self.cfg.getDataConfig("cache", "../cache/")
self.appfolder = self.cfg.getDataConfig("folder", "./appdata/")
self.outputfn = self.appfolder + "msdata_zack_" + datetime.datetime.now(
).strftime("%Y-%m-%d") + '.csv'
def __init__(self):
self.mtd = marketdata.MarketData()
self.sinaapi = SinaMarketData()
self.yahoofeed = FeederYahoo()
self.googfeed = FeederGoogle()
self.datacfg = ms_config.MsDataCfg("")
self.cachepath = self.datacfg.getDataConfig("cache", "../cache/")
pass
def __init__(self):
self.zack = data_zacks.data_zacks()
self.mtd = marketdata.MarketData()
self.cfg = ms_config.MsDataCfg("")
self.zackfile = self.cfg.getDataConfig("zack")
self.cachefolder = self.cfg.getDataConfig("folder")
BaseIndNoPx.__init__(self)
return
def __init__(self):
self.zack = data_zacks.data_zacks()
self.mtd = marketdata.MarketData()
self.cfg = ms_config.MsDataCfg("")
self.zackfile = self.cfg.getDataConfig("zack")
self.cachefolder = self.cfg.getDataConfig("cache", "../cache/")
self.hasCachedDF = False
BaseIndNoPx.__init__(self)
return
def __init__(self):
self.columns = [
'symbol', 'pricesale', 'marketcap', 'avgdailyvol', 'px'
]
self.colcode = "&f=sp5j1a2l1"
self.sufname = "fundaupdate_"
self.outputpath = "../data/"
self.mkt = marketdata.MarketData()
self.stgyInx = {}
def __init__(self):
self.mtd = marketdata.MarketData()
cfg = ms_config.MsDataCfg("")
symbolLstFile = cfg.getDataConfig("marketdata")
df = self.mtd.loadSymbolLstFile(symbolLstFile)
self.googdict = df.set_index('goog')['googexg'].to_dict()
'''
for key in self.googdict:
if (self.googdict[key]!=""):
print key,self.googdict[key]
'''
pass
def __init__(self):
pandas.set_option('display.max_columns', 100)
pandas.set_option('display.precision', 3)
pandas.set_option('display.expand_frame_repr', False)
#pandas.set_option('display.height', 1500)
pandas.set_option('display.max_rows', 1500)
self.mtd = marketdata.MarketData()
self.cfg = ms_config.MsDataCfg()
self.outputpath = "cache/"
self.outputfn = self.outputpath + "msdata_reuter_" + self.cfg.getCsvFileSurfix(
)
self.fileName = self.cfg.getDataConfig("marketdata",
"./marketdata.csv")
self.reuterFile = ""
self.option = ""
self.tickdf = pandas.DataFrame()
self.columns = [
'saleqtr0','saleqtr-1','saleqtr-2','saleqtr-3','saleqtr-4','saleqtr-5','saleqtr-6','saleqtr-7',\
'saleqtr-8','saleqtr-9','epsqtr0','epsqtr-1','epsqtr-2','epsqtr-3','epsqtr-4','epsqtr-5','epsqtr-6',\
'epsqtr-7','epsqtr-8','epsqtr-9',\
\
'numest','saleq1e','saleq1ey','saleq2e','saleq2ey','saley1e','saley1ey','saley2e','saley2ey','epsq1e',\
'epsq1ey','epsq2e','epsq2ey','epsy1e','epsy1ey','epsy2e','epsy2ey','numltgr','ltgre','ltgrey',\
\
'cppettm','indupettm','sectorpettm','cppehigh5y','indupehigh5y','sectorpehigh5y','cppelow5y','indupelow5y',\
'sectorpelow5y','cpbeta','indubeta','sectorbeta','cppsttm','indupsttm','sectorpsttm','cppbmrq','indupbmrq',\
'sectorpbmrq','cppcfttm','indupcfttm','sectorpcfttm','cppfcfttm','indupfcfttm','sectorpfcfttm',\
\
'divyield','div5y','div5ygr','payoutratio',\
\
'cpsalemrqyoy','indusalemrqyoy','sectorsalemrqyoy','cppsalettmyoy','indusalettmyoy','sectorsalettmyoy','cpsale5ygr',\
'indusale5ygr','sectorsale5ygr','cpepsmrqyoy','induepsmrqyoy','sectorepsmrqyoy','cpepsttmyoy','induepsttmyoy',\
'sectorepsttmyoy','cpeps5ygr','indueps5ygr','sectoreps5ygr','cpcap5ygr','inducap5ygr','sectorcap5ygr',\
\
'cpdebt2equity','indudebt2equity','secdebt2equity','cpquira','induquira','secquira','cpcurra','inducura','seccurra',\
\
'cpgm','indugm','sectorgm','cpgm5y','indugm5y','sectorgm5y','cpom','induom','sectorom','cpom5y','induom5y','sectorom5y',\
'cpnm','indunm','sectornm','cpnm5y','indunm5y','sectornm5y',\
\
'cproa','induroa','sectorroa','cproa5y','induroa5y','sectorroa5y','cproi','induroi','sectorroi','cproi5y','induroi5y','sectorroi5y','cproe','induroe',\
'sectorroe','cproe5y','induroe5y','sectorroe5y'
]
self.allcols = ['symbol', 'exg'] + self.columns
self.pid = 0
# base set columns
self.colbase = ['symbol','cppettm','cppsttm','cppbmrq','cppcfttm',\
'divyield','payoutratio',\
'cpsalemrqyoy','cppsalettmyoy','cpsale5ygr','cpepsmrqyoy','cpepsttmyoy','cpeps5ygr',\
'cpcurra','cpquira','cpdebt2equity',\
'cpgm','cpom','cpnm','cpbeta']
def main():
config = configparser.ConfigParser()
config.read('config.ini')
logging.basicConfig(level=logging.INFO)
## logging.basicConfig(level=logging.DEBUG)
past_market_data = marketdata.MarketData()
past_market_data.load_from_file()
past_market_data.clean()
actual_market_data = marketdata.MarketData()
actual_market_data.load_from_html(
config['DEFAULT']['Source is web'] == str(True))
past_market_data.append(actual_market_data.marketdata)
past_market_data.clean()
past_market_data.save_to_file()
# wait for key stroke
os.system('pause')
def __init__(self):
self.mkt = marketdata.MarketData()
self.funda = fundata.FundaData()
self.reuter = reuterfunda.ReuterFunda()
self.outputpath = "../result/"
self.symtable = pandas.DataFrame()
self.fundafile = ""
self.zackfile = ""
self.reuterfile = ""
self.screen = ""
self.enddate = datetime.datetime.now().strftime("%Y-%m-%d")
pandas.set_option('display.max_columns', 50)
pandas.set_option('display.precision', 3)
pandas.set_option('display.expand_frame_repr', False)
#pandas.set_option('display.height', 1500)
pandas.set_option('display.max_rows', 1500)
return
def __init__(self):
self.parameter = {}
self.stgyBatchCfg = []
#default log to file, unless specified in parameter
#sys.stdout = open("cbdaylog.txt", "w")
pandas.set_option('display.max_columns', 50)
pandas.set_option('display.precision', 3)
pandas.set_option('display.expand_frame_repr', False)
#pandas.set_option('display.height', 1500)
pandas.set_option('display.max_rows', 1500)
self.dataPath = "../data/"
self.resultPath = "../result/"
self.strategyPath = "../strategy/"
self.screenPath = "../screen/"
self.pid = 0 #dow30=0,focus=1
self.nmuBest = 3 # 3 best result
self.tradesup = tradesupport.Trade(self)
self.simutable = simutable.SimuTable(self)
self.mkt = marketdata.MarketData()
def __init__(self):
#self.rankPattern = '[\d\D]*Zacks.*Rank[/s]*: (.*) <sup class=[\d\D]*'
#self.rankPattern = '[\d\D]*Zacks.*Rank[/s]*: (.*)[\n]*<sup class=[\d\D]*'
#<p>Zacks Rank : 2-Buy <sup class=" AAPL
#self.rankPattern = '[\d\D]*Zacks[\D]*Rank[/s]*: (.)[\d\D]*'
self.rankPattern = '[\d\D]*Zacks[\D]*Rank[\s]?: (.)[\d\D]*'
self.cqestCol = ['cq0', 'cq7', 'cq30', 'cq60', 'cq90']
self.columns = ['rank','indurank','indutotal','etf','abrt','abr1w','abr1m','abr2m',\
'abr3m','numbr'] + self.cqestCol
self.allcols = ['symbol', 'exg'] + self.columns
self.pid = 0
self.outputfn = "msdata_zack_" + datetime.datetime.now().strftime(
"%Y-%m-%d") + '.csv'
self.option = ""
self.tickdf = pandas.DataFrame()
self.mtd = marketdata.MarketData()
self.cfg = ms_config.MsDataCfg() # default = datafile, cb_config.cfg
self.fileName = self.cfg.getDataConfig(
"", "marketdata") #"./marketdata.csv"
self.zackfile = self.cfg.getDataConfig("", "zack") #"./marketdata.csv"
def main():
config = configparser.ConfigParser()
config.read('config.ini')
## logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)
reg = registry.Registry()
market_data = marketdata.MarketData()
market_data.load_from_file()
### ??? ###
## slist = screener.ShortList()
## scr = screener.Screener()
## slist.assemble(scr)
##
## slist.save_to_file()
# wait for key stroke
os.system('pause')
def __init__(self):
pandas.set_option('display.max_columns', 50)
pandas.set_option('display.precision', 3)
pandas.set_option('display.expand_frame_repr', False) #expand wide dataframe
pandas.set_option('display.max_rows', 1500)
#pandas.set_option('display.width', 100)
#pandas.set_option('max_colwidth', 600)
#pandas.set_option('display.float_format', lambda x: '%.3f' % x)
self.outputpath = "../result/"
#self.mscfg = "./marketscan.cfg" #??
self.mtd = marketdata.MarketData()
self.mfeed = ms_feed.ms_feed()
self.csvchart = ms_csvchart.ms_csvchart()
self.params = ms_paramparser.ms_paramparser()
self.datacfg = ms_config.MsDataCfg("")
self.cachepath = self.datacfg.getDataConfig("folder","../cache/")
self.sgyInx={}
self.sp500 = "^GSPC" #?
self.nmuBest = 1 #??
return
def __init__(self):
self.reuter = reuterfunda.ReuterFunda()
self.reuterFile = "./msdata_reuter_2015-05-20.csv"
self.mtd = marketdata.MarketData()
return
def __init__(self):
#print "BaseIndNoPx init"
self.mtd = marketdata.MarketData()
self.cleanup()
def downloadInstruments(self, symbols, start_date, final_date):
if not os.path.exists("downloads"):
os.makedirs("downloads")
millis = int(round(time.time() * 1000))
downloadFileName = f'{str(millis)}.csv'
symbolsArray = symbols.split(',')
print(f'datasource ={self.datasource }')
print(f'symbolName ={self.symbolColumn}')
print(f'dateColumn ={self.dateColumn }')
print(f'priceColumn={self.priceColumn }')
print(f'date_format={self.date_format }')
print(f'symbols.split={symbolsArray}')
print(
f'start_date={start_date}, final_date={final_date}, downloadFileName={downloadFileName}'
)
session = requests.session()
session.headers = requests.utils.default_headers()
session.headers[
'Accept'] = 'text/html,application/json,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
session.headers['Accept-Encoding'] = 'gzip, deflate, br'
session.headers[
'Accept-Language'] = 'en,ru-RU;q=0.9,ru;q=0.8,en-US;q=0.7'
session.headers['Cache-Control'] = 'max-age=0'
session.headers['Connection'] = 'keep-alive'
session.headers['Upgrade-Insecure-Requests'] = '1'
session.headers[
'User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36'
session.headers['Content-Type'] = 'application/json'
session.headers['Authorization'] = 'Token {}'.format(self.access_key)
# print 'using headers v.6'
downloaders = {
"marketdata":
md.MarketData(symbolsArray,
start_date,
final_date,
session=session),
"converter_invest":
lambda: cv_invest.FormatConverter(start_date, self.inputfilename),
"converter_asxhist":
lambda: cv_asxhist.FormatConverter(start_date, self.inputfilename),
"converter_tiingo":
lambda: cv_tiingo_conv.FormatConverter(start_date, symbolsArray),
"tiingo":
lambda: tiingo.TiingoExt(symbolsArray,
start_date,
final_date,
api_key=self.access_key,
retry_count=10,
pause=0.3,
extheaders=session.headers),
"quandl":
lambda: qr.Quandl(symbolsArray,
start_date,
final_date,
api_key=self.access_key,
retry_count=10,
pause=0.3,
extheaders=session.headers),
"marketindex":
lambda: mi.MarketIndex(symbolsArray,
start_date,
final_date,
api_key=self.access_key,
retry_count=10,
pause=0.3,
extheaders=session.headers)
}
allColumnsOrigDf = downloaders[self.source]().read()
allColumnsNoIndexDf = allColumnsOrigDf.reset_index()
print(
f'allColumnsNoIndexDf column number: {len(allColumnsNoIndexDf.columns)}.'
f'\n columns: {allColumnsNoIndexDf.columns.values}')
print(f'Index from DataReader: {allColumnsNoIndexDf.index}')
if self.priceColumn in allColumnsNoIndexDf:
dataDf = allColumnsNoIndexDf[[
self.symbolColumn, self.dateColumn, self.priceColumn
]]
else:
print(f'Column {self.priceColumn} doesn'
't exist. Trying fallback for using ['
'Symbol'
', '
'Date'
', '
'Close'
']')
dataDf = allColumnsNoIndexDf[['Symbol', 'Date', 'Close']]
dataDf.to_csv('downloads/downloadedInstrumentsOnlyOnePriceColRaw.csv')
newDf = self.pivot(dataDf, start_date, final_date)
if not os.path.exists(self.directory):
os.makedirs(self.directory)
earliest_date = start_date + dt.timedelta(days=7)
newDf = newDf[newDf.index >= earliest_date]
existing_earliest_date = newDf.index.min()
# existing_earliest_date = newDf.loc(newDf.index >= earliest_date).index.min()
print(
f'earliest_date = {earliest_date}, existing_earliest_date = {existing_earliest_date}'
)
cols_to_exclude = newDf.loc[:existing_earliest_date].isnull().all()
cols_to_keep = ~cols_to_exclude
print(f'Earliest date to compare: {str(existing_earliest_date)}')
newDf = newDf.loc[:, cols_to_keep]
newDf.to_csv(self.directory + '/before_dropna_' + downloadFileName)
# newDf.dropna(axis=0, inplace=True)
newDf.dropna(axis=1, inplace=True)
newDf.to_csv(self.directory + '/' + downloadFileName)
print('instruments download finished')
return self.directory + '/' + downloadFileName
def __init__(self):
self.fileName = ""
self.mtd = marketdata.MarketData()
return
def __init__(self):
self.mtd = marketdata.MarketData()
self.datacfg = ms_config.MsDataCfg("")
self.cachepath = self.datacfg.getDataConfig("folder", "../cache/")
pass
class B3Parser:
md = marketdata.MarketData()
def get_futures(self, date_start, date_end=None, symbol=None):
print(str(date_start))
dstart = datetime.datetime.strptime(str(date_start), '%Y%m%d')
dend = None
if date_end != None:
dend = datetime.datetime.strptime(
str(date_end), '%Y%m%d')+datetime.timedelta(days=1) # soma um dia
if dend != None:
while dstart < dend:
self.md.get_futures(date=dstart.strftime('%Y%m%d'))
self.extract(dstart.strftime('%Y%m%d'), symbol)
dstart = dstart+datetime.timedelta(days=1) # soma um dia
def extract(self, date, symbol):
name = None
if os.path.exists(futpath+b3TRADE+date+'.zip') is False:
print("File does not exist!")
return -1
zf = zipfile.ZipFile(futpath+b3TRADE+date+'.zip')
name = zf.namelist()[0]
print(name)
zf.extract(name, futpath)
zf.close()
if symbol != None:
with open(futpath+name, 'r') as f:
head = next(f).split()
tradingRows = int(head[4].lstrip('0'))
print(tradingRows)
md = csv.DictReader(f, delimiter=';', fieldnames=headers)
line = 2
try:
os.stat(futpath+'/'+symbol)
except:
os.mkdir(futpath+'/'+symbol)
with open(futpath+'/'+symbol+'/'+date,'w') as out:
writer = csv.DictWriter(out, delimiter=';', extrasaction='ignore', fieldnames=headers)
writer.writeheader()
for row in md:
if line < tradingRows:
if symbol in row['Instrument Symbol']:
row['Instrument Symbol']=row['Instrument Symbol'].strip()
row['Trade Number']=row['Trade Number'].lstrip('0')
row['Trade Price']=row['Trade Price'].lstrip().lstrip('0')
row['Traded Quantity']=row['Traded Quantity'].lstrip('0')
row['Sequential Buy Order Number']=row['Sequential Buy Order Number'].lstrip('0')
row['Secondary Order ID - Buy Order']=row['Secondary Order ID - Buy Order'].lstrip('0')
row['Sequential Sell Order Number']=row['Sequential Sell Order Number'].lstrip('0')
row['Secondary Order ID - Sell Order']=row['Secondary Order ID - Sell Order'].lstrip('0')
row['Buy Member']=row['Buy Member'].lstrip('0')
row['Sell Member']=row['Sell Member'].lstrip('0')
writer.writerow(row)
# print(row['Instrument Symbol'])
line += 1
f.close()
os.remove(futpath+name)
def run(self):
pass
if __name__ == '__main__':
run()
