def crawl(url):
'''
Crawl entire site
:param url: URL of website
:type url: str
'''
if not url.startswith('http://') and not url.startswith('https://'):
url = 'http://%s/' % url
if newurl(url):
print "New URL"
else:
print "URL already in DB"
return
print url
# get correct title, detect language
p = Site(title=url, url=url, show=0)
p.published = datetime.datetime.today()
p.save()
thr = threading.Thread(target=__crawl_address, args=[url], kwargs={})
thr.start()
return
