def test_invalid_url(self):
self.assertFalse(is_valid_url("ftp://simon987.net"))
self.assertFalse(is_valid_url("git://simon987.net"))
self.assertFalse(is_valid_url("simon987.net"))
self.assertFalse(is_valid_url("http://simon987.net:8080"))
self.assertFalse(is_valid_url("http://simon987/"))
def try_enqueue(url):
url = os.path.join(url, "")
url = od_util.get_top_directory(url)
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the appropriate scheme.", "warning"
website = db.get_website_by_url(url)
if website:
return "Website already exists", "danger"
website = db.website_exists(url)
if website:
return "A parent directory of this url has already been posted", "danger"
if db.is_blacklisted(url):
return "<strong>Error:</strong> " \
"Sorry, this website has been blacklisted. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
if not od_util.is_od(url):
return "<strong>Error:</strong>" \
"The anti-spam algorithm determined that the submitted url is not " \
"an open directory or the server is not responding. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
website_id = db.insert_website(Website(url, str(request.remote_addr + "_" +
request.headers.get("X-Forwarded-For", "")),
request.user_agent))
task = Task(website_id, url, priority=1)
taskManager.queue_task(task)
return "The website has been added to the queue", "success"
def try_enqueue(url):
url = os.path.join(url, "")
url = od_util.get_top_directory(url)
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the appropriate scheme."
website = db.get_website_by_url(url)
if website:
return "Website already exists"
website = db.website_exists(url)
if website:
return "A parent directory of this url has already been posted"
if db.is_blacklisted(url):
return "<strong>Error:</strong> " \
"Sorry, this website has been blacklisted. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>."
if not od_util.is_od(url):
return "<strong>Error:</strong>" \
"The anti-spam algorithm determined that the submitted url is not " \
"an open directory or the server is not responding. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>."
website_id = db.insert_website(Website(url, "localhost", "mass_import.py"))
task = Task(website_id, url, priority=2)
taskManager.queue_task(task)
return "The website has been added to the queue"
def try_enqueue(url):
url = os.path.join(url, "")
website = db.get_website_by_url(url)
if website:
return "Website already exists", "danger"
website = db.website_exists(url)
if website:
return "A parent directory of this url has already been posted", "danger"
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the http(s):// suffix. " \
"FTP is not supported", "danger"
if od_util.is_blacklisted(url):
return "<strong>Error:</strong> " \
"Sorry, this website has been blacklisted. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
if not od_util.is_od(url):
return "<strong>Error:</strong>" \
"The anti-spam algorithm determined that the submitted url is not " \
"an open directory or the server is not responding. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
web_id = db.insert_website(
Website(url, str(request.remote_addr), str(request.user_agent)))
db.enqueue(web_id)
return "The website has been added to the queue", "success"
def test_valid_url(self):
self.assertTrue(is_valid_url("https://google.ca/"))
self.assertTrue(is_valid_url("http://google.ca/"))
self.assertTrue(is_valid_url("http://www.google.ca/"))
self.assertTrue(is_valid_url("http://www.subdomain.google.ca/"))
self.assertTrue(is_valid_url("http://mộtsốkýtựngẫunhiên.whatever/"))
self.assertTrue(is_valid_url("http://simon987.net:1234/"))
self.assertTrue(is_valid_url("http://simon987.net:12345/"))
# in progress
print(url)
print("In progress")
continue
handle_exact_repost(website.id, comment)
continue
website_id = db.website_exists(url)
if website_id:
if not scanned:
print("Parent in progress")
continue
handle_subdir_repost(website_id, comment)
continue
if not od_util.is_valid_url(url):
print("Skipping reddit comment: Invalid url")
bot.reply(
comment, "Hello, " + str(comment.author) +
". Unfortunately it seems that the link you "
"provided: `" + url +
"` is not valid. Make sure that you include the"
"'`http(s)://` prefix. \n")
continue
if od_util.is_blacklisted(url):
print("Skipping reddit comment: blacklisted")
bot.reply(
comment, "Hello, " + str(comment.author) +
". Unfortunately my programmer has "
"blacklisted this website. If you think that this is an error, please "
# in progress
print(url)
print("In progress")
continue
handle_exact_repost(website.id, comment)
continue
website_id = db.website_exists(url)
if website_id:
if not scanned:
print("Parent in progress")
continue
handle_subdir_repost(website_id, comment)
continue
if not od_util.is_valid_url(url):
print("Skipping reddit comment: Invalid url")
bot.reply(comment, "Hello, " + str(comment.author) + ". Unfortunately it seems that the link you "
"provided: `" + url + "` is not valid. Make sure that you include the"
"'`http(s)://` prefix. \n")
continue
if od_util.is_blacklisted(url):
print("Skipping reddit comment: blacklisted")
bot.reply(comment, "Hello, " + str(comment.author) + ". Unfortunately my programmer has "
"blacklisted this website. If you think that this is an error, please "
"[contact him](https://old.reddit.com/message/compose?to=Hexahedr_n)")
continue
if not od_util.is_od(url):
print("Skipping reddit comment: Not an OD")