def crawl_qr(counter): counter = counter.upper() slist = formStocklist(counter, S.KLSE_LIST) qrUrl = I3_QR_URL + slist[counter] + ".jsp" if S.DBG_ALL or S.DBG_QR: print ("\tQR: " + counter + " " + slist[counter] + " " + qrUrl) qr = scrape_qr(counter, slist[counter], connect_url(qrUrl)) return qr
def crawl_insider(counter, trading_date): stk_list = formStocklist(counter, S.KLSE_LIST) dirUrl = I3_INSIDER_DIRECTOR_URL + stk_list[counter] + ".jsp" shdUrl = I3_INSIDER_SHAREHOLDER_URL + stk_list[counter] + ".jsp" if S.DBG_ALL or S.DBG_INSIDER: print ("\tInsider: " + counter + " " + stk_list[counter] + " " + dirUrl) dirList = scrape_insider(counter, stk_list[counter], connect_url(dirUrl), trading_date) if S.DBG_ALL or S.DBG_INSIDER: print ("\tInsider: " + counter + " " + stk_list[counter] + " " + shdUrl) shdList = scrape_insider(counter, stk_list[counter], connect_url(shdUrl), trading_date) return dirList, shdList
def compileLinks(i3, sb, counters): i3chartlinks = [] sbchartlinks = [] SB_URL = 'https://my.stockbit.com/#/symbol/KLSE-' stocklist = formStocklist(counters, getI3Dir() + 'klse.txt') for key in stocklist.iterkeys(): if i3: i3chartlinks.append(S.I3_KLSE_URL + '/servlets/stk/chart/' + stocklist[key] + '.jsp') if sb: sbchartlinks.append(SB_URL + key + '/chartbit') return i3chartlinks, sbchartlinks
postUpdateProcessing() stocks = getCounters(args['COUNTER'], args['--klse'], args['--portfolio'], args['--watchlist'], False) ''' global klse, mt4date klse = "scrapers/i3investor/klse.txt" mt4date = args['--mt4'] if args['--klse']: # Full download using klse.txt # To do: a fix schedule to refresh klse.txt print "Scraping i3 stocks listing ..." writeStocksListing(klse) elif args['--price']: sname = args['COUNTER'][0].upper() slist = formStocklist(sname, klse) nums = args['--price'].split(",") ratio = float(nums[0]) / float(nums[1]) pricesplit(sname, slist[sname], float("{:.5f}".format((ratio)))) elif args['--KLSE']: scrapeKlseRelated('scrapers/investingcom/klse.idmap') elif mt4date is not None: mt4update(mt4date) else: if args['COUNTER']: stocks = args['COUNTER'][0].upper() else: stocks = retrieveCounters(args['--list']) S.DBG_ALL = False S.RESUME_FILE = True
] dt = datetime.strptime(ds, '%Y-%m-%d') self.append(dt, open_, high, low, close, volume) if S.DBG_ALL: print "YahooQuote:End" if __name__ == '__main__': S.RESUME_FILE = False if not S.RESUME_FILE: lastdt = S.ABS_START klse = "../i3investor/klse.txt" stocks = '' if len(stocks) > 0: # download only selected counters stocklist = formStocklist(stocks, klse) else: # Full download using klse.txt stocklist = loadKlseCounters(klse) cookie, crumb = getYahooCookie('https://uk.finance.yahoo.com/quote/AAPL/') for shortname in sorted(stocklist.iterkeys()): stock_code = stocklist[shortname] if len(stock_code) > 0: # OUTPUT_FILE = '../../data/yahoo/' + shortname + "." + stock_code + ".csv" OUTPUT_FILE = getDataDir( S.DATA_DIR) + 'yahoo/' + shortname + "." + stock_code + ".csv" if S.RESUME_FILE: lastdt = getLastDate(OUTPUT_FILE) if len(lastdt) == 0:
print outfile fh = open(outfile, "w") for key in sorted(rpt.iterkeys()): listing = key + ',' + ','.join(map(str, unpackListing(*(rpt[key])))) if S.DBG_ALL: print listing fh.write(listing + '\n') fh.close() ''' if __name__ == '__main__': loadCfg(getDataDir(S.DATA_DIR)) db = initKlseDB() stocks = 'LCTITAN' if db is not None: if len(stocks) > 0: stocklist = formStocklist(stocks, '../i3investor/klse.txt') else: stocklist = loadMap('../i3investor/klse.txt', ',') for stock in sorted(stocklist.iterkeys()): for rpt in WSJRPT.iterkeys(): for term in WSJTERM.iterkeys(): writeStocksFinancials(stock, term, rpt) closeKlseDB() else: print "No DB connection" pass
def mpvUpdateKlseRelated(): stocks = retrieveCounters("k") print "Updating KLSE related:", stocks stocklist = formStocklist(stocks, KLSE) for shortname in sorted(stocklist.iterkeys()): mvpUpdateMPV(shortname, stocklist[shortname])
stocklist = formStocklist(stocks, KLSE) for shortname in sorted(stocklist.iterkeys()): mvpUpdateMPV(shortname, stocklist[shortname]) if __name__ == '__main__': args = docopt(__doc__) cfg = loadCfg(S.DATA_DIR) DBG_ALL = True if args['--debug'] else False if args['COUNTER']: stocks = args['COUNTER'][0].upper() else: stocks = retrieveCounters(args['--list']) if args['--others']: stocklist = {} stocklist[stocks] = 0 else: if len(stocks): stocklist = formStocklist(stocks, KLSE) else: stocklist = loadKlseCounters(KLSE) for shortname in sorted(stocklist.iterkeys()): if shortname in S.EXCLUDE_LIST: print "Excludes: ", shortname continue if args['--generate']: generateMPV(shortname, stocklist[shortname]) else: mvpUpdateMPV(shortname, stocklist[shortname])