def __init__(self, sendObject):
try:
self.searchTopic = sendObject['topic']
self.selectedTable = sendObject['table']
self.relatedSearchObject = sendObject['relatedSearch'].split(',')
if len(self.relatedSearchObject) == 1 and len(
sendObject['relatedSearch']) > len(sendObject['topic']):
self.searchTopic = sendObject['relatedSearch']
self.databaseId = sendObject['rowId']
parsedURL = urlparse.urlparse(self.dataLink)
self.baseLink = parsedURL.scheme + '://' + parsedURL.netloc + '/'
queryArray = {
'search_query': self.searchTopic,
# 'search_sort':'video_date_uploaded',
'filters': 'video, month',
'lclk': 'video',
'page': 1
}
self.fetchNavigationUrls = []
self.visitedNavigationPageIndex = []
self.fetchNavigationUrls.append(self.dataLink + '?' +
urllib.urlencode(queryArray))
self.visitedNavigationPageIndex.append(1)
except Exception as e:
print 'Error : During initialization !!'
print e
try:
self.linkArray = []
self.existedLinkArray = []
self.nameArray = []
sqlite = sqlite3Db(self.BASE_DIR +
'/dataBases/trendingFeed.sqlite3')
sqlite.table = "`" + self.selectedTable + "`"
sqlite.fields = "`title`, `link`"
sqlite.where = '`topic_id`=' + str(self.databaseId)
tempData = sqlite.select()
sqlite.clear()
sqlite.close()
sqlite = ""
for data in tempData:
self.nameArray.append(data[0])
self.existedLinkArray.append(data[1])
self.totalVideoFoundCounter = len(self.nameArray)
except Exception as e:
print 'Error : During init database data fetch !!'
print e
def __init__(self, sendObject):
try:
self.searchTopic = sendObject['topic']
self.selectedTable = sendObject['table']
self.relatedSearchObject = sendObject['relatedSearch'].split(',')
if len(self.relatedSearchObject) == 1 and len(sendObject['relatedSearch']) > len(sendObject['topic']):
self.searchTopic = sendObject['relatedSearch']
self.databaseId = sendObject['rowId']
parsedURL = urlparse.urlparse(self.dataLink)
self.baseLink = parsedURL.scheme + '://' + parsedURL.netloc + '/'
queryArray = {
'search_query':self.searchTopic,
# 'search_sort':'video_date_uploaded',
'filters':'video, month',
'lclk':'video',
'page': 1
}
self.fetchNavigationUrls = []
self.visitedNavigationPageIndex = []
self.fetchNavigationUrls.append(self.dataLink + '?' + urllib.urlencode(queryArray))
self.visitedNavigationPageIndex.append(1)
except Exception as e:
print 'Error : During initialization !!'
print e
try:
self.linkArray = []
self.existedLinkArray = []
self.nameArray = []
sqlite = sqlite3Db(self.BASE_DIR + '/dataBases/trendingFeed.sqlite3')
sqlite.table = "`"+ self.selectedTable +"`"
sqlite.fields = "`title`, `link`"
sqlite.where = '`topic_id`='+ str(self.databaseId)
tempData = sqlite.select()
sqlite.clear()
sqlite.close()
sqlite = ""
for data in tempData:
self.nameArray.append(data[0])
self.existedLinkArray.append(data[1])
self.totalVideoFoundCounter = len(self.nameArray)
except Exception as e:
print 'Error : During init database data fetch !!'
print e
def saveDatainDatabase(self, dataObject, navId):
try:
sqlite = sqlite3Db(self.BASE_DIR + '/dataBases/dressAndTop.sqlite3')
sqlite.table = '`crawled_data`'
sqlite.columns = "`link`, `img_src`, `name`, `brand_name`, `price`, `discount_price`, `availability`, `navigation_id`"
sqlite.values = "?, ?, ?, ?, ?, ?, ?, ?"
sqlite.insertArray = [(dataObject['link'], dataObject['imgSrc'], dataObject['name'], dataObject['brandName'], dataObject['price'], dataObject['discountPrice'], True, navId)]
insertedId = sqlite.insert()
sqlite.close()
print colored('Log : Crawler data inserted ->'+ str(insertedId), 'green')
except Exception as e:
print colored('Error : during saveDatainDatabase function call !!', 'red')
print e
def insertData(self, inserDataArray): try: sqlite = sqlite3Db(self.BASE_DIR + '/dataBases/trendingFeed.sqlite3') sqlite.table = "`"+ self.selectedTable +"`" sqlite.columns = "`topic_id`, `title`, `description`, `link`, `meta_keywords`, `duration`, `img_thumbnail`, `views`, `publish_date`, `category`, `likes`, `dislikes`, `paid`, `channel_id`, `video_id`, `embed_url`, `family_friendly`, `region_allowed`" sqlite.values = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?" sqlite.insertManyArray = inserDataArray insertId = sqlite.insertMany() print 'Log : Youtube video inserted ->'+ str(insertId) sqlite.clear() sqlite.close() sqlite = "" self.totalVideoFoundCounter = self.totalVideoFoundCounter + 1 except Exception as e: print 'Error : Issue during database insertion !!' print e
def insertData(self, inserDataArray):
try:
sqlite = sqlite3Db(self.BASE_DIR +
'/dataBases/trendingFeed.sqlite3')
sqlite.table = "`" + self.selectedTable + "`"
sqlite.columns = "`topic_id`, `title`, `description`, `link`, `meta_keywords`, `duration`, `img_thumbnail`, `views`, `publish_date`, `category`, `likes`, `dislikes`, `paid`, `channel_id`, `video_id`, `embed_url`, `family_friendly`, `region_allowed`"
sqlite.values = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?"
sqlite.insertManyArray = inserDataArray
insertId = sqlite.insertMany()
print 'Log : Youtube video inserted ->' + str(insertId)
sqlite.clear()
sqlite.close()
sqlite = ""
self.totalVideoFoundCounter = self.totalVideoFoundCounter + 1
except Exception as e:
print 'Error : Issue during database insertion !!'
print e
def saveDatainDatabase(self, dataObject, navId):
try:
sqlite = sqlite3Db(self.BASE_DIR +
'/dataBases/dressAndTop.sqlite3')
sqlite.table = '`crawled_data`'
sqlite.columns = "`link`, `img_src`, `name`, `brand_name`, `price`, `discount_price`, `availability`, `navigation_id`"
sqlite.values = "?, ?, ?, ?, ?, ?, ?, ?"
sqlite.insertArray = [
(dataObject['link'], dataObject['imgSrc'], dataObject['name'],
dataObject['brandName'], dataObject['price'],
dataObject['discountPrice'], True, navId)
]
insertedId = sqlite.insert()
sqlite.close()
print colored('Log : Crawler data inserted ->' + str(insertedId),
'green')
except Exception as e:
print colored('Error : during saveDatainDatabase function call !!',
'red')
print e
