def start_agent(self, driver, UL,tag_list):
chrome_options = Options()
chrome_options.add_extension('Testings\SetupVPN - Lifetime Free VPN.crx')
driver = webdriver.Chrome(executable_path="C:\chromedriver4.exe", chrome_options=chrome_options)
driver.get(UL)
wait = random.uniform(0, 1)
sleep(wait)
sss = scrapeSingleSet()
sss.bycommontagid(UL,tag_list,driver)
def run(self):
single = scrapeSingleSet()
sequen = scrapeSequentialSets()
if (self.methodType == 1):
single.bytagid(self.url, self.tagList)
elif (self.methodType == 2):
single.bycommontagid(self.url, self.tagList)
elif (self.methodType == 3):
sequen.splitLines(self.url, self.dataset_id, self.tagList)
elif (self.methodType == 4):
sequen.bycommontagid(self.url, self.dataset_id, self.tagList)
else:
print "wrong method type given"
def startScrapingInThreads(url, dataset_id, tagList, methodType):
single = scrapeSingleSet()
sequen = scrapeSequentialSets()
if (methodType == 1):
result = single.bytagid(url, tagList)
elif (methodType == 2):
result = single.bycommontagid(url, tagList)
elif (methodType == 3):
result = sequen.splitLines(url, dataset_id, tagList)
elif (methodType == 4):
result = sequen.bycommontagid(url, dataset_id, tagList)
else:
print "wrong method type given"
return result