def getDataFromLast7Dayz(symbol):
isCreated = dbop.createTable(symbol)
print "getting data from last 7 days for ", symbol
msg = "getting data from last 7 days for " + symbol
Log(msg)
sc = Scrapper()
result = sc.equityScrapper(symbol, selected=False, timeout=100)
def getDataFromLastYears(symbol, historic=False):
# get the year from which we have to start scraping.
year = int(props.get("startYear"))
# check for company's db or create it if not created already.
isCreated = dbop.createTable(symbol, historic)
# this loop code will form dates and scrape data from the startYear say 2000 till last year's December say 2017.
while year < currentYear:
startMonth = 1
endMonth = startMonth + 1
while endMonth < 13:
if not historic:
result = formDateAndCallScrapper(startMonth, endMonth, year)
else:
result = formDateAndCallScrapper(startMonth,
endMonth,
year,
historic=True)
startMonth = endMonth + 1
endMonth = startMonth + 1
year += 1
startDay = 1
startMonth = 1
endMonth = startMonth + 1
limitMonth = int(datetime.datetime.now().strftime("%m")) # Current month
# now this loop is for the last slot of month/months which couldn't form 2 months pack.
while endMonth < limitMonth:
if not historic:
result = formDateAndCallScrapper(startMonth, endMonth, year)
else:
result = formDateAndCallScrapper(startMonth,
endMonth,
year,
historic=True)
startMonth = endMonth + 1
endMonth = startMonth + 1
if limitMonth - startMonth == 0 or limitMonth - startMonth == 1:
startDate = "0" + str(startDay) + "-0" + str(startMonth) + "-" + str(
year)
endDate = str(datetime.datetime.now().strftime("%d-%m-%Y"))
print "start - ", startDate, " to end - ", endDate
msg = "start - " + startDate + " to end - " + endDate
Log(msg)
if not historic:
sc = Scrapper()
result = sc.equityScrapper(symbol,
startDate,
endDate,
selected=True,
timeout=100)
else:
sc = Scrapper(historic=True)
result = sc.historicScrapper(startDate, endDate)
def formDateAndCallScrapper(startMonth, endMonth, year, historic=False):
dates = dt.dateCreator(startMonth, endMonth, year)
print "start - ", dates[0], " to end - ", dates[1]
msg = "start - " + dates[0] + " to end - " + dates[1]
Log(msg)
if not historic:
sc = Scrapper()
return sc.equityScrapper(symbol,
dates[0],
dates[1],
selected=True,
timeout=100)
else:
sc = Scrapper(historic=True)
return sc.historicScrapper(dates[0], dates[1])