def check_website():
"""Check to make sure websites are online.
If they're not change the settings.ini to say False.
Use a url that will make the website query and not a cached page."""
import time
config = configparser.RawConfigParser(allow_no_value=True)
config.read(settings['settings'])
websites = (dict(config.items('Websites')))
while True:
for website in websites:
browser = False
if website == "sankakucomplex":
ping_url = r"https://chan.sankakucomplex.com/?tags=1girl+rating%3Asafe+solo&commit=Search"
elif website == "safebooru":
ping_url = r"http://safebooru.org/index.php?page=post&s=list&tags=c.c.+1girl"
elif website == "danbooru":
continue
elif website == "yande":
continue
elif website == "konachan":
continue
browser = utils.scrape_site(ping_url)
if not browser or browser is False:
# Website is offline/timed out.
print("$ Website {0} has been set to offline!".format(website))
config_save('Websites', website, 'False', file=0)
else:
# Site is online. Turn it back on.
if websites[website] == "False":
print("$ Website {0} has been set to online!".format(website))
config_save('Websites', website, 'True', file=0)
time.sleep(5 * 60)
def check_website():
"""Check to make sure websites are online.
If they're not change the settings.ini to say False.
Use a url that will make the website query and not a cached page."""
import time
while True:
config = configparser.RawConfigParser(allow_no_value=True)
config.read(settings['settings'])
websites = (dict(config.items('Websites')))
for website in websites:
browser = False
if website == "sankakucomplex":
ping_url = (r"https://chan.sankakucomplex.com/"
"?tags=1girl+rating%3Asafe+solo&commit=Search")
elif website == "safebooru":
ping_url = (r"http://safebooru.org/index.php?"
"page=post&s=list&tags=c.c.+1girl")
elif website == "danbooru":
continue
elif website == "yande":
continue
elif website == "konachan":
continue
browser = utils.scrape_site(ping_url)
if not browser or browser is False:
# Website is offline/timed out.
print("$ Website {0} has been set to offline!".format(website))
config_save('Websites', website, 'False', file=0)
else:
# Site is online. Turn it back on.
if websites[website] == "False":
print("$ Website {0} has been set to online!".format(
website))
config_save('Websites', website, 'True', file=0)
time.sleep(5 * 60)
def get_soup(self, site):
if site == 0:
cookie_file = "sankakucomplex.txt"
url_search = "https://chan.sankakucomplex.com/?tags="
elif site == 2:
cookie_file = "safebooru.txt"
a = "http://safebooru.org/"
url_search = a + "index.php?page=post&s=list&tags="
tags = self.name + self.end_tags
if site == 0 or site >= 2:
tags += "+rating:safe"
url = url_search + tags
return scrape_site(url, cookie_file)
def get_soup(self, site):
if site == 0:
cookie_file = settings['secret_key'] + "-sankakucomplex.txt"
url_search = "https://chan.sankakucomplex.com/?tags="
elif site == 2:
cookie_file = settings['secret_key'] + "-safebooru.txt"
a = "http://safebooru.org/"
url_search = a + "index.php?page=post&s=list&tags="
tags = self.name + self.end_tags
if site == 0 or site > 2:
tags += "+rating:safe"
url = url_search + tags
return scrape_site(url, cookie_file)
def info(image):
url = "http://iqdb.org/?url=%s" % (str(image))
soup = utils.scrape_site(url)
if not soup:
return "OFFLINE", "OFFLINE", "OFFLINE"
if soup.find('th', text="No relevant matches"):
return False, False, False
site = None
links = soup.find_all('a')
for link in links:
try:
link['href']
except:
continue
if link.string == "(hide)":
# Haven't broke yet, onto low results
return False, False, False
if "chan.sankakucomplex.com/post/show/" in link['href']:
if "http" not in link['href']:
url = "http:" + link['href']
else:
url = link['href']
site = 0
break
elif "http://danbooru.donmai.us/posts/" in link['href']:
url = link['href']
site = 1
break
if site is None:
# No link found!
return False, False, False
soup = utils.scrape_site(url)
if not soup:
return "OFFLINE", "OFFLINE", "OFFLINE"
try:
if site == 0:
# Sankaku
artist = soup.find(
'li', class_="tag-type-artist").find_next('a').text.title()
elif site == 1:
# Danbooru
artist = soup.find(
'h2', text="Artist").find_next(
'a', class_="search-tag").text.title()
except:
artist = ""
try:
if site == 0:
# Sankaku
series = soup.find(
'li', class_="tag-type-copyright").find_next(
'a').text.title()
elif site == 1:
# Danbooru
series = soup.find(
'h2', text="Copyrights").find_next(
'a', class_="search-tag").text.title()
except:
series = ""
try:
if site == 0:
# Sankaku
names = soup.find_all('li', class_="tag-type-character")
elif site == 1:
# Danbooru
names = soup.find_all('li', class_="category-4")
name = []
for a in names:
if site == 0:
a = a.find_next('a').text.title()
elif site == 1:
a = a.find_next('a').find_next('a').text.title()
name.append(re.sub(r' \([^)]*\)', '', a))
name = list(set(name))
if len(name) >= 2:
names = utils.make_paste(
text='\n'.join(name),
title="Source Names")
else:
names = ''.join(name)
except:
names = ""
return artist, series, names
def airing(args):
if len(args) <= 3:
return False
args = args.replace("Durarara!!x2", "Durarara!!×2")
air_list_titles = []
air_list_msg = []
url = "https://www.livechart.me/summer-2015/all"
soup = utils.scrape_site(url)
if not soup:
return False
show_list = soup.find_all('h3', class_="main-title")
today = datetime.datetime.today()
today = today + datetime.timedelta(hours=8)
for div in show_list:
anime_title = div.text
if len(anime_title) > 60:
anime_title = anime_title[:50] + "[...]"
air_list_titles.append(anime_title)
ep_num_time = div.findNext('div').text
if "ished" in ep_num_time or \
"ished" in ep_num_time and "eatrical" in ep_num_time:
ep_num = "Finished"
elif "eatrical" in ep_num_time:
ep_num = "Movie"
else:
try:
if "EP" in ep_num_time:
ep_num = "Episode " + ep_num_time.split(":", 1)[0][2:]
else:
if "Movie" in anime_title:
ep_num = "Movie"
else:
ep_num = "Movie/OVA/Special"
except:
ep_num = "New Series"
if "Episode" in ep_num or "Movie" in ep_num:
try:
ep_num_time = ep_num_time.split(": ", 1)[1]
except:
pass
try:
ep_num_time = ep_num_time.split(
" JST", 1)[0].replace(" at", "")
anime_time = datetime.datetime.strptime(re.sub(
" +", " ", ep_num_time), '%b %d %I:%M%p')
anime_time = anime_time.replace(
year=datetime.datetime.today().year)
result = anime_time - today
msg = ("{0}\n"
"{1} airing in\n"
"{2} Days, {3} Hours and {4} Minutes").format(
anime_title, ep_num, result.days,
result.seconds//3600, (result.seconds//60) % 60)
except:
msg = "{0}\nNew Series\nUnknown air date!".format(anime_title)
else:
msg = "{0} has finished airing!".format(anime_title)
air_list_msg.append(msg)
try:
found = [s for s in air_list_titles if re.sub(
'[^A-Za-z0-9]+', '', args.lower()) in re.sub(
'[^A-Za-z0-9]+', '', s.lower())]
found = ''.join(found[0])
index = air_list_titles.index(''.join(found))
air = air_list_msg[index]
air = ''.join(air)
except:
count_trigger("airing")
return False
return air
def get_image_online(**kwargs):
if kwargs.get('used images'):
txt_name = kwargs.get('used images')
used_links = open(txt_name, 'r').read().splitlines()
else:
txt_name = os.path.join(os.getcwd(), "Used sankaku {0}.txt".format(
kwargs['bot name']))
try:
used_links = open(txt_name, 'r').read().splitlines()
except:
if not os.path.exists(txt_name):
print("Didn't find any used links! Creating a TXT!")
print("Set it to:\n{0}".format(txt_name))
used_links = []
else:
used_links = open(txt_name, 'r').read().splitlines()
if kwargs.get('highest page'):
high_page = int(kwargs.get('highest page'))
else:
high_page = 50
tried_pages = [high_page]
cookie_file = None
try_count = 0
low_page = 0
page = 0
x = None
no_images = False
url_start = "https://chan.sankakucomplex.com"
url_search = "https://chan.sankakucomplex.com/?tags="
if utils.is_bool(kwargs.get('login')):
cookie_file = "../sankakucomplex.txt"
url_login = "******"
form_num = 0
form_user = "******"
form_password = "******"
username = kwargs.get('username')
password = kwargs.get('password')
if not os.path.exists(cookie_file):
browser, s = utils.scrape_site(url_login, cookie_file, True)
form = browser.get_form(form_num)
form[form_user].value = username
form[form_password].value = password
browser.submit_form(form)
s.cookies.save()
if utils.is_bool(kwargs.get('save images')):
if kwargs.get('path'):
path = kwargs.get('path')
else:
path = os.path.abspath(os.path.join(os.getcwd(),
"images"))
if not os.path.exists(path):
os.makedirs(path)
else:
path = os.path.abspath(os.path.join(os.getcwd()))
if kwargs.get('tags'):
if isinstance(kwargs.get('tags'), list):
tags = '+'.join(kwargs.get('tags'))
else:
tags = '+'.join(kwargs.get('tags').split(', '))
else:
tags = ""
if kwargs.get('ignore tags'):
if isinstance(kwargs.get('ignore tags'), list):
ignore_tags = kwargs.get('ignore tags')
else:
ignore_tags = kwargs.get('ignore tags').split(', ')
else:
ignore_tags = []
if utils.is_bool(kwargs.get('ignore cosplay')):
ignore_cosplay = utils.is_bool(kwargs.get('ignore cosplay'))
else:
ignore_cosplay = False
if utils.is_bool(kwargs.get('accept webm')):
accept_webm = utils.is_bool(kwargs.get('accept webm'))
else:
accept_webm = False
tried_pages = [high_page + 1]
while True:
while True:
while True:
while True:
no_images = False
try_count += 1
if try_count == 15:
return False, False
page = str(int(random.randint(low_page, high_page) * 1))
while int(page) in tried_pages:
if int(page) == 0:
break
if not x:
x = high_page
page = str(int(
random.randint(low_page, high_page) * 1))
if int(page) > int(x):
continue
tried_pages.append(int(page))
x = min(tried_pages)
page_url = "&page=" + str(page)
url = "%s%s%s" % (url_search, tags, page_url)
browser = utils.scrape_site(url, cookie_file)
if browser.find('div', text="No matching posts"):
no_images = True
time.sleep(1)
if not no_images:
break
elif no_images and int(page) == 0:
return False, False
good_image_links = []
image_links = browser.find_all('a')
for link in image_links:
try:
link['href']
except:
continue
if "/post/show/" not in link['href']:
continue
good_image_links.append(link['href'])
if good_image_links == []:
return False, False
random.shuffle(good_image_links)
url = "%s%s" % (url_start, random.choice(good_image_links))
try_count = 0
while url in used_links:
url = "%s/%s" % (
url_start, random.choice(good_image_links))
try_count = try_count + 1
if try_count == 20:
break
used_links.append(url)
# Make a copy for better use in message
post_url = url
browser.open(url)
if not accept_webm:
if browser.find('video', attrs={'id': 'image'}):
continue
image_tags = []
char_tags = []
art_tags = []
sers_tags = []
tags_tags = []
site_tag = browser.find('ul', id="tag-sidebar")
site_tag = site_tag.find_all('li')
for taga in site_tag:
tag = tag_clean(taga)
if taga['class'][0] == "tag-type-artist":
art_tags.append(tag.title())
elif taga['class'][0] == "tag-type-copyright":
sers_tags.append(tag.title())
elif taga['class'][0] == "tag-type-character":
char_tags.append(tag.title())
else:
tags_tags.append(tag.title())
image_tags.append(tag.lower())
if any([item in [x.lower() for x in ignore_tags]
for item in [x.lower() for x in image_tags]]):
continue
if ignore_cosplay:
if any(" (cosplay)" in s for s in image_tags):
continue
break
image_url = browser.find('img', attrs={'id': 'image'})
if not image_url:
image_url = browser.find('video', attrs={'id': 'image'})
try:
url = urllib.parse.urljoin("https:", image_url['src'])
except:
# Flash File
continue
filename = ""
if not utils.is_bool(kwargs.get('message')):
message = ""
sn_kwgs = {}
sn_url, sn_kwgs = utils.saucenao(url, kwargs['saucenao api'], True)
re_dict = {'{#artist}': (
'#' if art_tags else '') + ' #'.join(
[x.replace(" ", "_") for x in art_tags]),
'{#character}': (
'#' if char_tags else '') + ' #'.join(
[x.replace(" ", "_") for x in char_tags]),
'{#series}': (
'#' if sers_tags else '') + ' #'.join(
[x.replace(" ", "_") for x in sers_tags]),
'{#tags}': (
'#' if tags_tags else '') + ' #'.join(
[x.replace(" ", "_") for x in tags_tags]),
'{artist}': ', '.join(art_tags),
'{character}': ', '.join(char_tags),
'{series}': ', '.join(sers_tags),
'{tags}': ', '.join(tags_tags),
'{url}': post_url,
'{title}': sn_kwgs.get('title'),
'{sn title}': sn_kwgs.get('title'),
'{sn illust id}': sn_kwgs.get('illust id'),
'{sn illust url}': sn_url,
'{sn artist}': sn_kwgs.get('artist'),
'{sn artist id}': sn_kwgs.get('artist id'),
'{sn artist url}': sn_kwgs.get('artist url')}
if kwargs.get('filename'):
filename = utils.replace_all(kwargs.get('filename'), re_dict)
filename = utils.safe_msg(filename)
if kwargs.get('message'):
message = utils.replace_all(kwargs.get('message'), re_dict)
message = utils.safe_msg(message)
with open(txt_name, 'w+') as f:
f.write("\n".join(used_links))
tweet_image = utils.download_image(url, path, filename, **kwargs)
if tweet_image:
break
if not utils.is_bool(kwargs.get('save images')):
from threading import Thread
Thread(name="Delete Image", target=delete_image, args=(
tweet_image, )).start()
return message, tweet_image
def airing(args):
if len(args) <= 3:
return False
args = args.replace("Durarara!!x2", "Durarara!!×2")
air_list_titles = []
air_list_msg = []
url = "https://www.livechart.me/summer-2015/all"
soup = utils.scrape_site(url)
if not soup:
return False
show_list = soup.find_all('h3', class_="main-title")
today = datetime.datetime.today()
today = today + datetime.timedelta(hours=8)
for div in show_list:
anime_title = div.text
if len(anime_title) > 60:
anime_title = anime_title[:50] + "[...]"
air_list_titles.append(anime_title)
ep_num_time = div.findNext('div').text
if "ished" in ep_num_time or \
"ished" in ep_num_time and "eatrical" in ep_num_time:
ep_num = "Finished"
elif "eatrical" in ep_num_time:
ep_num = "Movie"
else:
try:
if "EP" in ep_num_time:
ep_num = "Episode " + ep_num_time.split(":", 1)[0][2:]
else:
if "Movie" in anime_title:
ep_num = "Movie"
else:
ep_num = "Movie/OVA/Special"
except:
ep_num = "New Series"
if "Episode" in ep_num or "Movie" in ep_num:
try:
ep_num_time = ep_num_time.split(": ", 1)[1]
except:
pass
try:
ep_num_time = ep_num_time.split(
" JST", 1)[0].replace(" at", "")
anime_time = datetime.datetime.strptime(re.sub(
" +", " ", ep_num_time), '%b %d %I:%M%p')
anime_time = anime_time.replace(
year=datetime.datetime.today().year)
result = anime_time - today
msg = ("{0}\n"
"{1} airing in\n"
"{2} Days, {3} Hours and {4} Minutes").format(
anime_title, ep_num, result.days,
result.seconds//3600, (result.seconds//60) % 60)
except:
msg = "{0}\nNew Series\nUnknown air date!".format(anime_title)
else:
msg = "{0} has finished airing!".format(anime_title)
air_list_msg.append(msg)
try:
found = [s for s in air_list_titles if re.sub(
'[^A-Za-z0-9]+', '', args.lower()) in re.sub(
'[^A-Za-z0-9]+', '', s.lower())]
found = ''.join(found[0])
index = air_list_titles.index(''.join(found))
air = air_list_msg[index]
air = ''.join(air)
except:
count_trigger("airing")
return False
return air
def info(image):
url = "http://iqdb.org/?url=%s" % (str(image))
soup = utils.scrape_site(url)
if not soup:
return "OFFLINE", "OFFLINE", "OFFLINE"
if soup.find('th', text="No relevant matches"):
return False, False, False
site = None
links = soup.find_all('a')
for link in links:
try:
link['href']
except:
continue
if link.string == "(hide)":
# Haven't broke yet, onto low results
return False, False, False
if "chan.sankakucomplex.com/post/show/" in link['href']:
if "http" not in link['href']:
url = "http:" + link['href']
else:
url = link['href']
site = 0
break
elif "http://danbooru.donmai.us/posts/" in link['href']:
url = link['href']
site = 1
break
if site is None:
# No link found!
return False, False, False
soup = utils.scrape_site(url)
if not soup:
return "OFFLINE", "OFFLINE", "OFFLINE"
try:
if site == 0:
# Sankaku
artist = soup.find(
'li', class_="tag-type-artist").find_next('a').text.title()
elif site == 1:
# Danbooru
artist = soup.find(
'h2', text="Artist").find_next(
'a', class_="search-tag").text.title()
except:
artist = ""
try:
if site == 0:
# Sankaku
series = soup.find(
'li', class_="tag-type-copyright").find_next(
'a').text.title()
elif site == 1:
# Danbooru
series = soup.find(
'h2', text="Copyrights").find_next(
'a', class_="search-tag").text.title()
except:
series = ""
try:
if site == 0:
# Sankaku
names = soup.find_all('li', class_="tag-type-character")
elif site == 1:
# Danbooru
names = soup.find_all('li', class_="category-4")
name = []
for a in names:
if site == 0:
a = a.find_next('a').text.title()
elif site == 1:
a = a.find_next('a').find_next('a').text.title()
name.append(re.sub(r' \([^)]*\)', '', a))
name = list(set(name))
if len(name) >= 2:
names = utils.make_paste(
text='\n'.join(name),
title="Source Names")
else:
names = ''.join(name)
except:
names = ""
return artist, series, names
def get_reddit(**kwargs):
if kwargs.get('used images'):
txt_name = kwargs.get('used images')
used_links = open(txt_name, 'r').read().splitlines()
else:
txt_name = os.path.join(os.getcwd(), "Used reddit {0}.txt".format(
kwargs['bot name']))
try:
used_links = open(txt_name, 'r').read().splitlines()
except:
if not os.path.exists(txt_name):
print("Didn't find any used links! Creating a TXT!")
print("Set it to:\n{0}".format(txt_name))
used_links = []
else:
used_links = open(txt_name, 'r').read().splitlines()
try:
sub = used_links[0]
used_links = used_links[1:]
except:
# Probably doesn't exist (i hope only that)
pass
if kwargs.get('save images'):
if kwargs.get('path'):
path = kwargs.get('path')
else:
path = os.path.abspath(os.path.join(os.getcwd(),
"images"))
if not os.path.exists(path):
os.makedirs(path)
else:
path = os.path.abspath(os.path.join(os.getcwd()))
start_url = "https://www.reddit.com/r/"
subreddits = kwargs.get('subreddits')
is_random = kwargs.get('random subreddit')
is_random_link = kwargs.get('random link')
if subreddits is None:
return False, False
if isinstance(subreddits, str):
subreddits = subreddits.split(", ")
if utils.is_bool(is_random):
import random
sub = random.choice(subreddits)
else:
# Get last used sub and + 1
try:
sub = open(os.path.join(os.getcwd(), "Used reddit {0}.txt".format(
kwargs['bot name'])),
'r').read().splitlines
sub = subreddits[(subreddits.index(sub) + 1)]
except:
# Doesn't exsist / end of list
sub = subreddits[0]
url = start_url + sub + "/.rss"
soup = utils.scrape_site(url, is_rss=True)
pic_imgs = []
for a in soup.find_all('item'):
img_string = a.find('description').string
img_title = a.find('title').string
img_link = a.find('link').string
img_string = img_string[:img_string.index("[link]")]
img_string = BeautifulSoup(img_string, 'html5lib').find_all('a')
for item in img_string:
if "reddit.com" not in item['href'] and "http" in item['href']:
pic_imgs.append([item['href'], img_title, img_link])
if utils.is_bool(is_random_link):
import random
image = random.choice(pic_imgs)
else:
image = pic_imgs[0]
safe_break = 0
count = 0
while image[0] in used_links:
if utils.is_bool(is_random_link):
image = random.choice(pic_imgs)
else:
image = pic_imgs[count]
if image[0] in used_links:
count += 1
continue
break
safe_break += 1
if safe_break == 50:
break
used_links.append(image[0])
imgTypes = {"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
"webm": "video/webm"}
filepath = urlparse(image[0]).path
ext = os.path.splitext(filepath)[1].lower()
if not ext[ext.rfind(".") + 1:] in imgTypes:
if "imgur" in image[0]:
# Just make it .png it still returns correct image
image[0] = "http://i.imgur.com/" + image[0].rsplit(
'/', 1)[1] + ".png"
ext = ".png"
sn_kwgs = {}
if "(x-post" in image[1].lower() or "(via" in image[1].lower():
image[1] = re.sub(r'\([^)]*\)', '', image[1])
if "sn" in kwargs.get('message'):
sn_url, sn_kwgs = utils.saucenao(fname=image[0],
api_key=kwargs.get('saucenao api'),
metainfo=True)
re_dict = {'{url}': image[2],
'{title}': image[1],
'{sn title}': sn_kwgs.get('title'),
'{sn illust id}': sn_kwgs.get('illust id'),
'{sn illust url}': sn_url,
'{sn artist}': sn_kwgs.get('artist'),
'{sn artist id}': sn_kwgs.get('artist id'),
'{sn artist url}': sn_kwgs.get('artist url')}
if kwargs.get('filename'):
filename = utils.replace_all(kwargs.get('filename'), re_dict)
filename = utils.safe_msg(filename)
else:
filename = ""
if kwargs.get('message'):
message = utils.replace_all(kwargs.get('message'), re_dict)
message = utils.safe_msg(message)
else:
message = ""
image = utils.download_image(image[0], path, filename, **kwargs)
used_links = [sub] + used_links
with open(txt_name, 'w') as f:
f.write("\n".join(used_links))
return message, image
