def scrape(symbol): myDB = DBHelper() url = "https://www.google.com/finance/getprices?q=%s&x=NSE&i=60&p=1d&f=d,c,o,h,l,v&df=cpct&auto=1" %symbol req = urllib2.Request(url) req.add_header('User-Agent', 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0') try: f = urllib2.urlopen(req) rawdata = f.read() f.close() except Exception as e: logging.error("Error fetching URL") f.close() base_timestamp = 0 priceTuple = namedtuple('priceTuple', ['close', 'high', 'low', 'open', 'volume']) for idx, line in enumerate(rawdata.splitlines()): if (idx == 3): interval = int(line.strip().split('=')[1]) if (idx >= 7): content = line.strip().split(',') if (content[0].startswith('a')): timestamp = datetime.datetime.fromtimestamp(int(content[0][1:])) base_timestamp = timestamp p = priceTuple._make(tuple(content[1:])) else: timestamp = base_timestamp + datetime.timedelta(seconds=int(content[0]) * interval) p = priceTuple._make(tuple(content[1:])) try: myDB.insertPrice(timestamp=timestamp, symbol=symbol, pricetuple=p) except Exception as e: logging.error(e.message) conn = myDB.getConn() conn.commit() conn.close()
def run(): financeapi = GoogleFinanceAPI() while 1: myDB = DBHelper() scripsData = myDB.getAll(tbname='scrips') conn = myDB.getConn() conn.close() scripList = [scrip[0] for scrip in scripsData] chunkList = list(chunks(scripList,99)) #since URL can only be 2000 characters long and google finance returns only 100 symbols at a time for chunk in chunkList: time.sleep(1) quotelist = financeapi.get(chunk) myDB.updatelivedata(quotelist) time.sleep(2)
def run(): logging.basicConfig(filename='scyther.log', level= logging.DEBUG, format= '%(asctime)s %(message)s') logging.info('Backfill Started for %s at %s' %(datetime.datetime.now().date(),datetime.datetime.now())) myDB = DBHelper() scripsData = myDB.getAll(tbname = 'scrips') conn = myDB.getConn() conn.close() scripList = [scrip[0] for scrip in scripsData] idx = 0 for scrip in scripList: scrape(scrip) print idx, scrip time.sleep(1) idx += 1 logging.info('%s Scraped Successfully' %scrip) logging.info('Scraping completed successfully for %s' %datetime.datetime.now().date())
import psycopg2 from dbhelper import DBHelper mydb = DBHelper() scrips = mydb.getAll(tbname="scrips") conn = mydb.getConn() conn.close() scripList = [scrip[0] for scrip in scrips] mydb.insertlivedata(scripList)