def crawl_qr(counter):
counter = counter.upper()
slist = formStocklist(counter, S.KLSE_LIST)
qrUrl = I3_QR_URL + slist[counter] + ".jsp"
if S.DBG_ALL or S.DBG_QR:
print ("\tQR: " + counter + " " + slist[counter] + " " + qrUrl)
qr = scrape_qr(counter, slist[counter], connect_url(qrUrl))
return qr
def crawl_insider(counter, trading_date):
stk_list = formStocklist(counter, S.KLSE_LIST)
dirUrl = I3_INSIDER_DIRECTOR_URL + stk_list[counter] + ".jsp"
shdUrl = I3_INSIDER_SHAREHOLDER_URL + stk_list[counter] + ".jsp"
if S.DBG_ALL or S.DBG_INSIDER:
print ("\tInsider: " + counter + " " + stk_list[counter] + " " + dirUrl)
dirList = scrape_insider(counter, stk_list[counter], connect_url(dirUrl), trading_date)
if S.DBG_ALL or S.DBG_INSIDER:
print ("\tInsider: " + counter + " " + stk_list[counter] + " " + shdUrl)
shdList = scrape_insider(counter, stk_list[counter], connect_url(shdUrl), trading_date)
return dirList, shdList
def compileLinks(i3, sb, counters):
i3chartlinks = []
sbchartlinks = []
SB_URL = 'https://my.stockbit.com/#/symbol/KLSE-'
stocklist = formStocklist(counters, getI3Dir() + 'klse.txt')
for key in stocklist.iterkeys():
if i3:
i3chartlinks.append(S.I3_KLSE_URL + '/servlets/stk/chart/' +
stocklist[key] + '.jsp')
if sb:
sbchartlinks.append(SB_URL + key + '/chartbit')
return i3chartlinks, sbchartlinks
postUpdateProcessing()
stocks = getCounters(args['COUNTER'], args['--klse'],
args['--portfolio'], args['--watchlist'], False)
'''
global klse, mt4date
klse = "scrapers/i3investor/klse.txt"
mt4date = args['--mt4']
if args['--klse']:
# Full download using klse.txt
# To do: a fix schedule to refresh klse.txt
print "Scraping i3 stocks listing ..."
writeStocksListing(klse)
elif args['--price']:
sname = args['COUNTER'][0].upper()
slist = formStocklist(sname, klse)
nums = args['--price'].split(",")
ratio = float(nums[0]) / float(nums[1])
pricesplit(sname, slist[sname], float("{:.5f}".format((ratio))))
elif args['--KLSE']:
scrapeKlseRelated('scrapers/investingcom/klse.idmap')
elif mt4date is not None:
mt4update(mt4date)
else:
if args['COUNTER']:
stocks = args['COUNTER'][0].upper()
else:
stocks = retrieveCounters(args['--list'])
S.DBG_ALL = False
S.RESUME_FILE = True
]
dt = datetime.strptime(ds, '%Y-%m-%d')
self.append(dt, open_, high, low, close, volume)
if S.DBG_ALL:
print "YahooQuote:End"
if __name__ == '__main__':
S.RESUME_FILE = False
if not S.RESUME_FILE:
lastdt = S.ABS_START
klse = "../i3investor/klse.txt"
stocks = ''
if len(stocks) > 0:
# download only selected counters
stocklist = formStocklist(stocks, klse)
else:
# Full download using klse.txt
stocklist = loadKlseCounters(klse)
cookie, crumb = getYahooCookie('https://uk.finance.yahoo.com/quote/AAPL/')
for shortname in sorted(stocklist.iterkeys()):
stock_code = stocklist[shortname]
if len(stock_code) > 0:
# OUTPUT_FILE = '../../data/yahoo/' + shortname + "." + stock_code + ".csv"
OUTPUT_FILE = getDataDir(
S.DATA_DIR) + 'yahoo/' + shortname + "." + stock_code + ".csv"
if S.RESUME_FILE:
lastdt = getLastDate(OUTPUT_FILE)
if len(lastdt) == 0:
print outfile
fh = open(outfile, "w")
for key in sorted(rpt.iterkeys()):
listing = key + ',' + ','.join(map(str, unpackListing(*(rpt[key]))))
if S.DBG_ALL:
print listing
fh.write(listing + '\n')
fh.close()
'''
if __name__ == '__main__':
loadCfg(getDataDir(S.DATA_DIR))
db = initKlseDB()
stocks = 'LCTITAN'
if db is not None:
if len(stocks) > 0:
stocklist = formStocklist(stocks, '../i3investor/klse.txt')
else:
stocklist = loadMap('../i3investor/klse.txt', ',')
for stock in sorted(stocklist.iterkeys()):
for rpt in WSJRPT.iterkeys():
for term in WSJTERM.iterkeys():
writeStocksFinancials(stock, term, rpt)
closeKlseDB()
else:
print "No DB connection"
pass
def mpvUpdateKlseRelated():
stocks = retrieveCounters("k")
print "Updating KLSE related:", stocks
stocklist = formStocklist(stocks, KLSE)
for shortname in sorted(stocklist.iterkeys()):
mvpUpdateMPV(shortname, stocklist[shortname])
stocklist = formStocklist(stocks, KLSE)
for shortname in sorted(stocklist.iterkeys()):
mvpUpdateMPV(shortname, stocklist[shortname])
if __name__ == '__main__':
args = docopt(__doc__)
cfg = loadCfg(S.DATA_DIR)
DBG_ALL = True if args['--debug'] else False
if args['COUNTER']:
stocks = args['COUNTER'][0].upper()
else:
stocks = retrieveCounters(args['--list'])
if args['--others']:
stocklist = {}
stocklist[stocks] = 0
else:
if len(stocks):
stocklist = formStocklist(stocks, KLSE)
else:
stocklist = loadKlseCounters(KLSE)
for shortname in sorted(stocklist.iterkeys()):
if shortname in S.EXCLUDE_LIST:
print "Excludes: ", shortname
continue
if args['--generate']:
generateMPV(shortname, stocklist[shortname])
else:
mvpUpdateMPV(shortname, stocklist[shortname])