def search_db(keywords, limit=200):
sql = (
" SELECT * FROM `hindu` WHERE match(`title`, `link`) against('%s' IN BOOLEAN MODE) ORDER BY `id` desc LIMIT %d;"
% (keywords, limit))
query = Query()
res = query.search(sql)
query.close()
return res
def start_requests(self):
query = Query()
query.create()
query.close()
start_date = datetime.date(year=2017, month=2, day=1)
end_date = datetime.date(year=2017, month=3,
day=19) # datetime.date.today()
urls = []
for date in self.daterange(start_date, end_date):
self.date = date
date_formatted = date.strftime('%Y/%m/%d/')
url = self.base_url + date_formatted
yield scrapy.Request(url=url, callback=self.parse)
def insert(self, data):
sql = """INSERT INTO `hindu`(`title`,
`link`, `date`)
VALUES """
db_date = self.date.strftime('%Y-%m-%d')
f = open('/home/atul/sql.txt', 'w')
for row in data:
sql += '(\'' + row[0] + '\'' + ', ' + '\'' + row[
1] + '\'' + ', ' + '\'' + db_date + '\'),'
sql = sql.rstrip(',')
#f.write(sql)
query = Query()
query.execute(sql)
query.close()