def ImportProcessedURLs(self, urls_list, target_id=None):
for url, visited, scope in urls_list:
self.db.session.merge(
models.Url(target_id=target_id,
url=url,
visited=visited,
scope=scope))
self.db.session.commit()
def ImportURLs(self, url_list, target_id=None):
"""
Extracts and classifies all URLs passed. Expects a newline separated
URL list
"""
imported_urls = []
for url in url_list:
if self.IsURL(url):
imported_urls.append(url)
self.db.session.merge(models.Url(url=url, target_id=target_id))
self.db.session.commit()
return imported_urls # Return imported urls
def AddURLToDB(self, url, visited, found=None, target_id=None):
if self.IsURL(url): # New URL
# Make sure URL is clean prior to saving in DB, nasty bugs
# can happen without this
url = url.strip()
scope = self.target.IsInScopeURL(url)
self.db.session.merge(
models.Url(target_id=target_id,
url=url,
visited=visited,
scope=scope))
self.db.session.commit()