def try_enqueue(url):
url = os.path.join(url, "")
url = od_util.get_top_directory(url)
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the appropriate scheme.", "warning"
website = db.get_website_by_url(url)
if website:
return "Website already exists", "danger"
website = db.website_exists(url)
if website:
return "A parent directory of this url has already been posted", "danger"
if db.is_blacklisted(url):
return "<strong>Error:</strong> " \
"Sorry, this website has been blacklisted. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
if not od_util.is_od(url):
return "<strong>Error:</strong>" \
"The anti-spam algorithm determined that the submitted url is not " \
"an open directory or the server is not responding. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
website_id = db.insert_website(Website(url, str(request.remote_addr + "_" +
request.headers.get("X-Forwarded-For", "")),
request.user_agent))
task = Task(website_id, url, priority=1)
taskManager.queue_task(task)
return "The website has been added to the queue", "success"
def try_enqueue(url):
url = os.path.join(url, "")
url = od_util.get_top_directory(url)
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the appropriate scheme."
website = db.get_website_by_url(url)
if website:
return "Website already exists"
website = db.website_exists(url)
if website:
return "A parent directory of this url has already been posted"
if db.is_blacklisted(url):
return "<strong>Error:</strong> " \
"Sorry, this website has been blacklisted. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>."
if not od_util.is_od(url):
return "<strong>Error:</strong>" \
"The anti-spam algorithm determined that the submitted url is not " \
"an open directory or the server is not responding. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>."
website_id = db.insert_website(Website(url, "localhost", "mass_import.py"))
task = Task(website_id, url, priority=2)
taskManager.queue_task(task)
return "The website has been added to the queue"
def try_enqueue(url):
url = os.path.join(url, "")
website = db.get_website_by_url(url)
if website:
return "Website already exists", "danger"
website = db.website_exists(url)
if website:
return "A parent directory of this url has already been posted", "danger"
if not od_util.is_valid_url(url):
return "<strong>Error:</strong> Invalid url. Make sure to include the http(s):// suffix. " \
"FTP is not supported", "danger"
if od_util.is_blacklisted(url):
return "<strong>Error:</strong> " \
"Sorry, this website has been blacklisted. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
if not od_util.is_od(url):
return "<strong>Error:</strong>" \
"The anti-spam algorithm determined that the submitted url is not " \
"an open directory or the server is not responding. If you think " \
"this is an error, please <a href='/contribute'>contact me</a>.", "danger"
web_id = db.insert_website(
Website(url, str(request.remote_addr), str(request.user_agent)))
db.enqueue(web_id)
return "The website has been added to the queue", "success"
"provided: `" + url +
"` is not valid. Make sure that you include the"
"'`http(s)://` prefix. \n")
continue
if od_util.is_blacklisted(url):
print("Skipping reddit comment: blacklisted")
bot.reply(
comment, "Hello, " + str(comment.author) +
". Unfortunately my programmer has "
"blacklisted this website. If you think that this is an error, please "
"[contact him](https://old.reddit.com/message/compose?to=Hexahedr_n)"
)
continue
if not od_util.is_od(url):
print("Skipping reddit comment: Not an OD")
print(url)
bot.reply(
comment, "Hello, " + str(comment.author) +
". Unfortunately it seems that the link you "
"provided: `" + url +
"` does not point to an open directory. This could also"
" mean that the website is not responding (in which case, feel free to retry in "
"a few minutes). If you think that this is an error, please "
"[contact my programmer](https://old.reddit.com/message/compose?to=Hexahedr_n)"
)
continue
web_id = db.insert_website(
Website(url, "localhost", "reddit_bot"))
from database import Database
import od_util
import datetime
db = Database("db.sqlite3")
websites_to_update = db.get_websites_older(datetime.timedelta(minutes=5))
if websites_to_update:
for website_id in websites_to_update:
website = db.get_website_by_id(website_id)
# Ignore if website is down
if od_util.is_od(website.url):
# If website is still up, re-scan it
print("Re-scanning " + str(website_id))
db.clear_website(website_id)
db.enqueue(website_id)
else:
print("Website is down")
from database import Database
import od_util
import datetime
db = Database("db.sqlite3")
websites_to_update = db.get_websites_older(datetime.timedelta(minutes=5))
if websites_to_update:
for website_id in websites_to_update:
website = db.get_website_by_id(website_id)
# Ignore if website is down
if od_util.is_od(website.url):
# If website is still up, re-scan it
print("Re-scanning " + str(website_id))
db.clear_website(website_id)
db.enqueue(website_id)
else:
print("Website is down")
if not od_util.is_valid_url(url):
print("Skipping reddit comment: Invalid url")
bot.reply(comment, "Hello, " + str(comment.author) + ". Unfortunately it seems that the link you "
"provided: `" + url + "` is not valid. Make sure that you include the"
"'`http(s)://` prefix. \n")
continue
if od_util.is_blacklisted(url):
print("Skipping reddit comment: blacklisted")
bot.reply(comment, "Hello, " + str(comment.author) + ". Unfortunately my programmer has "
"blacklisted this website. If you think that this is an error, please "
"[contact him](https://old.reddit.com/message/compose?to=Hexahedr_n)")
continue
if not od_util.is_od(url):
print("Skipping reddit comment: Not an OD")
print(url)
bot.reply(comment, "Hello, " + str(comment.author) + ". Unfortunately it seems that the link you "
"provided: `" + url + "` does not point to an open directory. This could also"
" mean that the website is not responding (in which case, feel free to retry in "
"a few minutes). If you think that this is an error, please "
"[contact my programmer](https://old.reddit.com/message/compose?to=Hexahedr_n)")
continue
web_id = db.insert_website(Website(url, "localhost", "reddit_bot"))
db.enqueue(web_id, reddit_comment_id=comment.id, priority=2) # Medium priority for reddit comments
print("Queued comment post: " + str(web_id))
# Check posts
