def __runDetector(self, id, url, is_new):
craigslistSite = requests.get(url)
search = bs4.BeautifulSoup(craigslistSite.text, 'html.parser')
links = search.find_all("a")
matchFound = False
for link in links:
textFormat = str(link)
if ("data-id=" not in textFormat or "sandiego" not in textFormat):
continue
matchFound = True
beginningIndex = textFormat.find("href=") + 6
endingIndex = textFormat.find(".html") + 5
hyperlink = textFormat[beginningIndex:endingIndex]
if (is_new == True):
inputData = {}
inputData["hyperlink"] = hyperlink
inputData["keywords"] = id
inputData["is_deleted"] = False
if Url.find_url_by_hyperlink(inputData['hyperlink'],
inputData['keywords']):
return
Url(**inputData).save()
if not matchFound:
DetectorManager.__job_scheduler.add_job(
self.__runDetector,
'date',
run_date=self.__getRuntime(),
args=(id, url, is_new))
