class ImageRetriever:
def __init__(self):
self.logger = getLogger(__name__)
self.client = ImgurClient(getenv('IMGUR_CLIENT_ID'),
getenv('IMGUR_CLIENT_SECRET'))
def retrieve_images(self, no_pages=10):
images = filter(
lambda item: isinstance(item, GalleryImage) and item.animated,
reduce(add,
(self.client.gallery(page=i) for i in range(no_pages))))
pipe = app.db.pipeline()
pipe.delete('images')
for image in images:
print("Storing {}".format(image.id))
pipe.hsetnx('images', image.id, dumps(image))
pipe.execute()
def get_images(self):
return list(map(loads, app.db.hvals('images')))
def get_image(self, image_id):
try:
return loads(app.db.hget('images', image_id))
except TypeError:
raise KeyError()
async def imgur_meme(message, bot):
clientId = 'b49ec8d9045f4ff'
clientSecret = 'ff33b374c6c0faf7c591020cf621afdd963d1ea4'
try:
imager = ImgurClient(clientId, clientSecret)
randPage = random.randint(0, 10)
await bot.send_message(message.channel, "Fetching a spicy one...")
items = imager.gallery(page=randPage)
while True:
randImage = random.randint(0, len(items))
try:
if str(items[randImage].images[0]['link'])[-4:] == '.gif':
continue
await bot.send_message(message.channel, "```" + items[randImage].title + "```")
await bot.send_message(message.channel, items[randImage].images[0]['link'])
return
except AttributeError:
if str(items[randImage].link)[-4:] == '.gif':
continue
await bot.send_message(message.channel, "```" + items[randImage].title + "```")
await bot.send_message(message.channel, items[randImage].link)
return
except JSONDecodeError:
continue
except ImgurClientError:
continue
except CancelledError as e:
await bot.send_message(message.channel, "Imgur Cancelled Error: let Skittlely know")
print(e)
except TimeoutError as e:
await bot.send_message(message.channel, "Imgur Timeout Error: let Skittlely know")
print(e)
class ImageRetriever:
def __init__(self):
self.logger = getLogger(__name__)
self.client = ImgurClient(
getenv('IMGUR_CLIENT_ID'), getenv('IMGUR_CLIENT_SECRET'))
def retrieve_images(self, no_pages=10):
images = filter(
lambda item: isinstance(item, GalleryImage) and item.animated,
reduce(add, (self.client.gallery(page=i) for i in range(no_pages)))
)
pipe = app.db.pipeline()
pipe.delete('images')
for image in images:
print("Storing {}".format(image.id))
pipe.hsetnx('images', image.id, dumps(image))
pipe.execute()
def get_images(self):
return list(map(loads, app.db.hvals('images')))
def get_image(self, image_id):
try:
return loads(app.db.hget('images', image_id))
except TypeError:
raise KeyError()
def Random_Pic():
client = ImgurClient(client_id, client_secret)
items = client.gallery()
num = random.randint(0, len(items))
return items[num].link
async def imgur(message, client):
"""
Imgur stuff
:param message:
:param client:
:return:
"""
im_client_id = "c8093a713f38daa"
im_client_secret = "69a6863b617b86d562c6d45e45d9daf9134c283b"
im_client = ImgurClient(im_client_id, im_client_secret)
params = message.content.split()
paramslength = int(len(params))
try:
items = im_client.gallery(section='hot', sort='viral', page=0, window='day', show_viral=True)
get = int(params[2])
for num in range(0, get):
for item in items:
await client.send_message(message.channel, item.link)
except ImgurClientError as e:
print(e.error_message)
print(e.status_code)
def get_imgur_links(client_id, client_secret):
try:
client = ImgurClient(client_id, client_secret)
except ImgurClientError:
get_imgur_links(client_id, client_secret)
items = client.gallery()
return [x.link for x in items if x is not None]
def random_image():
load_dotenv()
client_id = os.getenv('IMGUR_TOKEN')
client_secret = os.getenv('IMGUR_SECRET')
client = ImgurClient(client_id, client_secret)
items = client.gallery()
item = random.choice(items)
return item.link
def BikiniBottomTwitter():
load_dotenv()
client_id = os.getenv('IMGUR_TOKEN')
client_secret = os.getenv('IMGUR_SECRET')
client = ImgurClient(client_id, client_secret)
items = client.gallery(section='r',
sort='BikiniBottomTwitter',
show_viral=False)
item = random.choice(items)
return item.link
def get_meme():
try:
client = ImgurClient(ImgurClientID, ImgurClientSecret)
except BaseException as e:
print('[EROR] Error while authenticating with Imgur:', str(e))
return
items = client.gallery()
list = []
for item in items:
list.append(item.link)
return random.choice(list)
def getImageLinks(pagenum=1, max_links=50):
#initialize client
client = ImgurClient(imgur_client_id, imgur_client_secret)
#Get items
items = client.gallery(section=CURRENT_SUBJECT, sort='time', page=pagenum,
window='day', show_viral=False)
#Get links
links = [item.link for item in items]
#Get image titles
titles = [item.title for item in items]
#Return list of links
return links[:max_links], titles[:max_links]
def imgur(bot, chat_id):
client_id = config.IMGUR_ID
client_secret = config.IMGUR_SECRET
client = ImgurClient(client_id, client_secret)
items = []
for i in range(0, 3):
items += client.gallery(section='hot',
sort='viral',
page=i,
window='day')
item = items[random.randrange(0, len(items))]
bot.send_photo(chat_id,
photo=item.link,
caption=item.title,
reply_markup=markup)
def getImageLinks(pagenum=1, max_links=50):
#initialize client
client = ImgurClient(imgur_client_id, imgur_client_secret)
#Get items
items = client.gallery(section=CURRENT_SUBJECT,
sort='time',
page=pagenum,
window='day',
show_viral=False)
#Get links
links = [item.link for item in items]
#Get image titles
titles = [item.title for item in items]
#Return list of links
return links[:max_links], titles[:max_links]
def get_imgur_image(text):
client_id = os.environ.get('IMGUR_CLIENT_ID')
client_secret = os.environ.get('IMGUR_CLIENT_SECRET')
if client_id and client_secret:
client = ImgurClient(client_id, client_secret)
tag = tag_from_text(text)
if tag:
try:
gallery = client.gallery_tag(tag)
gallery = gallery.items
except ImgurClientError:
return 'Tag {} not found'.format(tag)
else:
gallery = client.gallery()
return random_link_from_gallery_list(gallery)
def imgur(request, section='user', sort='rising', page=0):
if request.method == "POST":
form = GrabberForm(request.POST)
if form.is_valid():
post = form.save(commit=False)
post.time_added = datetime.now()
post.downloaded = False
post.save()
return redirect('post_detail', pk=post.pk)
else:
form = GrabberForm()
page = int(page)
client_id = getattr(settings, 'IMGUR_CLIENT_ID', '')
client_secret = getattr(settings, 'IMGUR_CLIENT_TOKEN', '')
client = ImgurClient(client_id, client_secret)
current_limits = client.credits
items = client.gallery(section=section,
sort=sort,
page=page,
show_viral=True)
album_images = {}
form = GrabberForm()
# images = client.get_album_images(items[1].id)
# Grab all the images in album
for item in items:
if item.is_album is True:
id = item.id
images = client.get_album_images(item.id)
album_images[id] = images
# album_images[id] = "test"
return render(
request, 'webapp/dashboard.html', {
'items': items,
'current_limits': current_limits,
'album_images': album_images,
'section': section,
'sort': sort,
'page': page,
'form': form,
})
def index():
client = ImgurClient(app.config['IMGUR_ID'], app.config['IMGUR_SECRET'])
try:
albums = client.gallery(sort='top', window='week', page=0)
except ConnectionError:
logger.error('Connection error')
return
images = list(islice((a for a in albums if matches_reqs(a)), 0, 10))
for img in images:
if img.account_id:
author = User.query.get(img.account_id)
if not author:
author = User(id=img.account_id, nickname=img.account_url)
db.session.add(author)
if not Album.query.get(img.id):
a = Album(id=img.id, link=img.link, author=author, title=img.title)
db.session.add(a)
db.session.commit()
return render_template(
"index.html",
images=images)
async def imgurCommand(message):
imgr = ImgurClient(IMGUR_ID, IMGUR_SECRET)
cmd = message.content.split(" ", 1)
if len(cmd) <= 1:
items = imgr.gallery(section='user',
sort='time',
page=1,
window='day',
show_viral=False)
else:
items = imgr.gallery_search(cmd[1],
advanced=None,
sort='time',
window='all',
page=0)
if len(items) < 1:
await message.channel.send("No reults")
i = random.randint(
0,
len(items) -
1) # Added -1 to prevent list index out of range exceptions
await message.channel.send(items[i].link)
def send_email():
genericLogger.info('Started send_email task')
client = ImgurClient(settings.IMGUR['CLIENT_ID'],
settings.IMGUR['CLIENT_SECRET'])
try:
gallery = client.gallery(sort='top', window='week', page=0)
except ConnectionError:
genericLogger.error('Connection error')
return
images = islice((item for item in gallery if matches_reqs(item)), 0, 10)
for s in Subscriber.objects.all():
genericLogger.info('Started user {}'.format(s.email))
email_content = generate_email(settings.HOST, s.name, s.email, images)
try:
send_mail(subject="Enjoy Most Popular ImgUr Images for the Last Week",
message="",
from_email='*****@*****.**',
recipient_list=[s.email],
html_message=email_content)
genericLogger.info('Sent email to {}'.format(s.email))
except SMTPException as e:
# invalid email ?
genericLogger.error(e)
from imgurpython import ImgurClient
client_id = 'YOUR CLIENT ID'
client_secret = 'YOUR CLIENT SECRET'
client = ImgurClient(client_id, client_secret)
# Example request
items = client.gallery()
for item in items:
print(item.link+",")
# Go th https://imgur.com/removalrequest to delete the image you upload
from imgurpython import ImgurClient
import random
client_id = ""
client_secret = ""
client = ImgurClient(client_id, client_secret)
links = []
items = client.gallery(section="/r/wallpapers", sort="top",
window="all") #gets image links from wallpaper section
for item in items:
links.append(item.link) #put all links on an array
def returnID(): # get a random link from links array and returns it
x = random.randrange(0, len(links))
return links[x]
def sms_reply():
# Gets the SMS sent to the number
body = request.values.get('Body', "")
check = command_check()
resp = MessagingResponse()
needshelp = 0
# !echo
if check == 1:
body = body[6:]
# Checks for parameters
if body == None:
resp.message("!echo requires additional input.")
else:
resp.message(body)
# Echoes
return str(resp)
# !wiki
elif check == 2:
body = body[6:]
# Checks for parameters
if body == None:
resp.message("!wiki requires additonal input.")
return str(resp)
# Search for article
try:
search = wikipedia.search(body, results=1, suggestion=True)
except wikipedia.exceptions.PageError as e:
resp.message("Error generating Wikipedia page.")
return str(resp)
# Removes disambiguation error
try:
article = wikipedia.page(title=search[0])
except wikipedia.exceptions.DisambiguationError as e:
article = wikipedia.page(title=e.options[0])
# Returns article info
title = article.title
summary = wikipedia.summary(title, sentences=3)
url = article.url
# Submits article info
wiki_stuff = title + "\n\n" + summary + "\n\n" + "Find out more at:" + url
resp.message(wiki_stuff)
return str(resp)
# !stock
elif check == 3:
body = body[7:]
# Checks for parameters
if body == None:
resp.message("!stock requires additonal input.")
body = body.upper()
stocklink = "https://finance.google.com/finance?q=" + body + "&output=json"
rsp = requests.get(stocklink)
if rsp.status_code in (200,):
fin_data = json.loads(rsp.content[6:-2].decode('unicode_escape'))
openprice = ('Opening Price: ${}'.format(fin_data['op']))
stockname = ('Stock Name: {}'.format(fin_data['name']))
stockinfo = stockname + "\n" + "Symbol: " + body + "\n" + openprice
if format(fin_data['op']) == "":
stockinfo = "Invalid stock symbol \n\nSymbols: Facebook(FB), Apple(AAPL), Alphabet(GOOG), Yahoo(YHOO), Amazon(AMZN), Coca-Cola(KO), Walmart(WMT), Microsoft(MSFT)\n\nMore at: https://finance.yahoo.com/"
resp.message(stockinfo)
return str(resp)
# !help
elif check == 0:
return help()
# !rand
elif check == 4:
body = body[6:]
# If no parameter is applied
if body == None:
items = ImgurClient.gallery()
randnum = random(0, len(items)-1)
img = get_image(items[randnum].image_id)
return img
else:
items = ImgurClient.gallery_search(q=body, advanced=None, sort='time', window='all', page=0)
randnum = random(0, len(items)-1)
img = get_image(items[randnum].image_id)
return img
# 5 Failed Commands = !help
else:
needshelp = needshelp + 1
if needshelp >= 5:
needshelp = 0
help()
return
from imgurpython import ImgurClient
from client_data import get_id, get_secret
client_id = get_id()
client_secret = get_secret()
client = ImgurClient(client_id, client_secret)
# Example request
items = client.gallery()
for item in items:
#print(item.link)
pass
def test_test(number):
return number + 10
class ImgUr(object):
"""
@Params: {string} - Download Directory
@Returns: None
"""
def __init__(self, dd=None):
client_id = '50049400837.73078400742'
client_secret = '5a730c87aeea8c2daad196ef3ff2c6da'
if not dd:
self.download_dir = 'downloads'
else:
self.download_dir = dd
self.setup_download_dir(self.download_dir)
self.data = []
self.client = ImgurClient(client_id, client_secret)
self.http = urllib3.PoolManager()
"""
@Params: {int} - Minimum number of images to download
@Description:
This method uses the ImgurClient to download pages of images from imgur and
places them into a list.
@Returns: {list} - List of urls to images
"""
def get_links(self, min=10):
page = 0
while len(self.data) <= min:
items = self.client.gallery(page=page,
window='day',
show_viral=True)
for item in items:
name, ext = os.path.splitext(item.link)
if ext == '.jpg' or ext == '.png':
self.data.append(item.link)
page += 1
return self.data
"""
@Params: {string} - Url of image to download
@Description:
This method receives a url and downloads it using the urllib3 library. Saves it to
a directory determined by the class constructor.
@Returns: None
"""
def download_link(self, link):
path = self.download_dir + "/" + os.path.basename(link)
r = self.http.request('GET', link, preload_content=False)
with open(path, 'wb') as out:
while True:
data = r.read()
if not data:
break
out.write(data)
r.release_conn()
"""
@Params: {string} - Download directory name
@Description:
This will check to see if the param is a directory and if not
it will create it.
@Returns: None
"""
def setup_download_dir(self, d):
self.download_dir = d
if not os.path.isdir(self.download_dir):
os.makedirs(self.download_dir)
png_count = 0
gif_count = 0
def countImage( type ):
global jpeg_count
global png_count
global gif_count
if(type == "image/jpeg"):
jpeg_count += 1
elif(type == "image/png"):
png_count += 1
elif(type == "image/gif"):
gif_count += 1
# Crawling Requests
items = client.gallery(page=0)
for item in items:
print(item.link)
if(not item.is_album):
countImage(item.type)
else:
for image in client.get_album_images(item.id):
countImage(image.type)
total_count = jpeg_count + png_count + gif_count
print("JPEG:" + str(jpeg_count) + " " + str(round(100 * (float(jpeg_count) / float(total_count)), 2)) + "%")
print("PNG:" + str(png_count) + " " + str(round(100 * (float(png_count) / float(total_count)), 2)) + "%")
print("GIF:" + str(gif_count) + " " + str(round(100 * (float(gif_count) / float(total_count)), 2)) + "%")
print("Total:" + str(total_count))
client = ImgurClient(client_id, client_secret, access_token, refresh_token)
# url = "https://api.imgur.com/3/gallery/top/time/day/2?showViral=false&mature=false&album_previews=false"
# headers = {'Authorization': 'Client-ID 9b315a3d4a73278'}
# response = requests.request("GET", url, headers=headers)
# print(response)
# res = requests.get('https://api.imgur.com/3/gallery/section/sort/window/page?showViral=false&mature=false&album_previews=false', params={'section': 'top', 'sort': 'viral', 'window': 'week', 'page': 3})
# print(res)
# Extracts the items (images) on the front page of Imgur:
# PROBLEM = I'm able to get the links, but not make it appear on the page with a Jinja template.
items = client.gallery(section='hot', sort='viral', page=2, window='week', show_viral=False)
for item in items:
print(item.link)
# print(item.title)
# print(items)
# create jinja for loop to iterate through each one to make a card
# Find the image on the front page that has the highest number of views
# items = client.gallery()
# max_item = None
# max_views = 0
# for item in items:
# if item.views > max_views:
# max_item = item
# max_views = item.views
# print(max_item.title)
class ImgurSpider(Spider):
name = 'imgur'
imgur = None
def setup_filter(self, sf):
sf.add_com('post', info='A post.')
sf.add_com('comment', info='A comment in reply to a post.')
sf_post = sf.get('post')
sf_post.add('title', info='The title of the post.')
sf_post.add('description', info='The description of the post.')
sf_post.add('author', info='The user who submitted the post.')
sf_post.add('points', info='The points of the post.')
sf_post.add('score', info='The score of the post.')
sf_post.add('ups', info='The number upvotes the post get.')
sf_post.add('downs', info='The number of downvotes the post get.')
sf_post.add('views', info='The view count of the post.')
sf_post.add('tags', info='The tags labelled on the post.')
sf_post.add('nsfw', info='States whether or not the post is NSFW.')
sf_comment = sf.get('comment')
sf_comment.add('body', info='The body of the comment.')
sf_comment.add('author', info='The user who submitted the comment.')
sf_comment.add('points', info='The points of the comment.')
sf_comment.add('ups', info='The upvotes the comment get.')
sf_comment.add('downs', info='The downvotes the comment get.')
sf.add_var(
'sections', type=list, default=['hot', 'new'],
info="Determine which sections posts are going to be "\
"retrieved from. The 'hot' section is labelled as "\
"'most viral' "\
"in imgur.com, and the 'new' section is labelled as "\
"'user submitted'"
)
sf.add_var(
'time', type=str,
choice=['day', 'week', 'month',\
'year', 'all'],
default='day',
info="The time frame of when the post was submitted. Note that "\
"this only applies to a 'hot' section with 'top' sorting. "
)
sf.add_var(
'sort', type=str,
choice=['popularity', 'top', 'newest'],
default='popularity',
info="Choose the order in how the posts are to be scanned. "\
"Note that the 'top' choice can be configured further "\
"with the @time attribute."
)
sf.add_var('skip_comments',
type=bool,
default=False,
info='Skip scanning over comments for every scanned post.')
sf.add_var(
'page_limit', type=int, default=999999,
info="The limit on how many page will the spider scan before it "\
"stops. The default is -1 (no limit)."
)
def start(self, sf):
client_id = '93814a7ab6dccf6'
client_secret = 'c9ed8ffe67e553f10f8d587f5d335ae68264fd92'
self.imgur = ImgurClient(client_id, client_secret)
for item in self.generator(sf):
yield item
def generator(self, sf):
"""
An item list generator containing the items in a page.
"""
sf_sections = sf.ret('sections')
sf_time = sf.ret('time')
sf_sort = sf.ret('sort')
p = 0
for section in sf_sections:
while True:
if p >= sf.ret('page_limit'):
return
page = self.imgur.gallery(section=section,
sort=sf_sort,
page=p,
window=sf.ret('time'))
p += 1
for post in self.generate_post(sf, page):
yield post
def generate_post(self, sf, page):
for post in page:
sf.get('post').set_attr_values(
author=post.account_url,
title=post.title,
description=post.description,
views=post.views,
points=int(post.points),
score=int(post.score),
ups=int(post.ups),
downs=int(post.downs),
tags=[tag['name'] for tag in post.tags],
nsfw=bool(post.nsfw))
if sf.get('post').should_scrape():
yield ComponentLoader(
'post', {
'post_id': post.id,
'author': post.account_url,
'title': post.title,
'description': post.description,
'link': post.link,
'topic': post.topic,
'points': post.points,
'score': post.score,
'ups': post.ups,
'downs': post.downs,
'comment_count': post.comment_count,
'tags': ','.join([tag['name'] for tag in post.tags]),
'nsfw': post.nsfw
})
if sf.ret('skip_comments'):
continue
for comment in self.generate_comment(sf, post.id):
yield comment
def generate_comment(self, sf, post_id):
for comment in self.imgur.gallery_item_comments(post_id):
sf.get('comment').set_attr_values(body=comment.comment,
author=comment.author,
points=comment.points,
ups=comment.ups,
downs=comment.downs)
if sf.get('comment').should_scrape():
yield ComponentLoader(
'comment', {
'comment_id': comment.id,
'parent_id': comment.parent_id,
'body': comment.comment,
'author': comment.author,
'datetime': comment.datetime,
'deleted': comment.deleted,
'points': comment.points,
'vote': comment.vote,
'downs': comment.downs,
'ups': comment.ups,
})
class Imgur:
"""The most awesome images on the internet!
This plugin allows basic searching on Imgur including subreddits.
Warning: Searching on subreddits cannot be easily moderated, therefore it is
extremely easy for a user to post images from an nsfw subreddit to whatever
channel the bot is enabled in if this plugin is enabled without modification.
The subreddit command can be disabled by changing 'enabled=True' to 'enabled=False'
in the plugin's main file: 'plugins/imgur.py' on line 53.
"""
def __init__(self, bot):
self.bot = bot
self.client = ImgurClient(config["client_id"], config["client_secret"])
@c.group(pass_context=True)
async def imgur(self, ctx):
"""Search on Imgur!"""
if ctx.invoked_subcommand is None:
await self.bot.say("Zoinks! You've taken a wrong turn! Try `help imgur`.")
# Helper function to actually get/post the images
async def post_image(self, request, query=None):
case = {
"subreddit": lambda: self.client.subreddit_gallery(query),
"search" : lambda: self.client.gallery_search(query),
"random" : lambda: self.client.gallery_random(),
"top" : lambda: self.client.gallery("top"),
"hot" : lambda: self.client.gallery("hot"),
"rehost" : lambda: self.client.upload_from_url(query),
}
function = case.get(request, None)
image = self.bot.loop.run_in_executor(None, function)
while True:
await asyncio.sleep(0.25)
if image.done():
image = image.result()
break
if request == "rehost":
await self.bot.say(image.get("link", None))
else:
await self.bot.say(random.choice(image).link)
@imgur.command(name="sub", aliases=["subreddit", "reddit", "r/"], enabled=True)
async def imgur_subreddit(self, subreddit: str):
"""Get an image from a subreddit."""
await self.post_image("subreddit", subreddit)
@imgur.command(name="search")
async def imgur_search(self, *, query: str):
"""Search Imgur for (almost) anything."""
await self.post_image("search", query)
@imgur.command(name="random")
async def imgur_random(self):
"""One free random image."""
await self.post_image("random")
@imgur.command(name="viral")
async def imgur_viral(self, section: str = "top"):
"""Get one of the most viral images of the day.
Section may be either 'top' or 'hot' and will get an image based on that criteria."""
section = section.lower()
if section != "top":
section = "hot"
await self.post_image(section)
@imgur.command(name="rehost")
async def imgur_rehost(self, url: str):
"""Rehost an image from any link to Imgur."""
await self.post_image("rehost", url)
import discord
import random
import asyncio
from discord.ext import commands
from imgurpython import ImgurClient
import os
#clients or defining variables#
client_id = os.environ['client_id']
client_secret = os.environ['client_secret']
grab = ImgurClient(
client_id, client_secret
) #had to give it a diffrent call to function from client as Discords prefix utilises "client" call to function#
items = grab.gallery()
class SFWCog:
def __init__(self, client):
self.client = client
async def on_ready(self):
print('SFWCog is Prepared')
#Imgur commands#
#memes and gifs#
@commands.command(
pass_context=True,
description=
"used for testing JSON file; can be used as .code_test **subtypename**"
import json
import os
from random import shuffle
with open('twitter_keys.txt') as json_file:
keys = json.load(json_file)
while True:
try:
#Imgur
client_id = keys['imgur'][0]['client_id']
client_secret = keys['imgur'][0]['client_secret']
client = ImgurClient(client_id, client_secret)
items=client.gallery(window='hour')
shuffle(items)
#tags_list = []
keyWords=['politics', 'vote', 'capitalism','trump', 'current events', 'republican', 'hatchact', 'election', 'fucktrump']
keyWords2 = ['funny', 'memes']
index = 0
size = len(items)
while index < size:
import datetime as dt
import requests
import csv
import sys
extract_date = dt.datetime.now()
#import config file and read in the values
config = configparser.ConfigParser()
config.read('auth.ini')
client_id = config.get('credentials', 'client_id')
client_secret = config.get('credentials', 'client_secret')
#open connection to imgur and retrieve all gallery images
client = ImgurClient(client_id, client_secret)
images = client.gallery()
image_info = []
image_tags = []
print('Processing.', end='')
#go through the gallery images and record the necessary fields; images and albums are treated differently
for image in images:
print('.', end='')
sys.stdout.flush()
if image.is_album == True:
#record fields for this album
image_info.append([
image.id, image.title, 'Album', image.views, image.ups,
image.downs, image.points, image.comment_count,
class ImgurRepostBot():
def __init__(self):
os.system('cls')
self.failed_downvotes = [] # Store failed downvotes for later processing
self.failed_comments = [] # Store failed comments for later processing
self.delay_between_requests = 5 # Changed on the fly depending on remaining credits and time until reset
self.thread_lock = threading.Lock()
self.logger = None
self.detected_reposts = 0
self.config = ConfigManager()
self._setup_logging()
self.imgur_client = ImgurClient(self.config.api_details['client_id'],
self.config.api_details['client_secret'],
self.config.api_details['access_token'],
self.config.api_details['refresh_token'])
self.db_conn = ImgurRepostDB(self.config)
self.backfill_progress = 1 if self.config.backfill else 'Disabled'
records, processed_ids = self.db_conn.build_existing_ids()
if self.config.backfill:
threading.Thread(target=self._backfill_database, name='Backfill').start()
self.hash_processing = HashProcessing(self.config, processed_ids, records)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
def _check_thread_status(self):
"""
Check status of critical threads. If they are found dead start them back up
:return:
"""
thread_names = ['configmonitor', 'backfill', 'repostprocessing']
for thrd in threading.enumerate():
if thrd.name.lower() in thread_names:
thread_names.remove(thrd.name.lower())
for i in thread_names:
if i == 'configmonitor':
msg = 'Config Monitor Thread Crashed'
self._output_error(msg)
self.config = ConfigManager()
continue
if i == 'backfill' and self.config.backfill:
msg = 'Backfill Thread Crashed'
self._output_error(msg)
threading.Thread(target=self._backfill_database, name='Backfill').start()
continue
if i == 'repostprocessing':
msg = 'Repost Processing Thread Crashed'
self._output_error(msg)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
continue
def _setup_logging(self):
if self.config.logging:
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.ERROR)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
fhandle = logging.FileHandler('botlog.log')
fhandle.setFormatter(formatter)
self.logger.addHandler(fhandle)
def _output_error(self, msg, output=True):
"""
convenience method to log and/or print an error
:param msg: Message to output/log
:param output: Print error to console
:return:
"""
if output:
print(msg)
if self.config.logging:
self.logger.error(msg)
def _output_info(self, msg):
print(msg)
if self.config.logging:
self.logger.info(msg)
def _backfill_database(self):
"""
Backfill the database with older posts. Useful if script hasn't been run in some time"
:return:
"""
while True:
if self.config.backfill:
original_start_page = self.config.backfill_start_page # So we can detect if it's changed in the config
current_page = self.config.backfill_start_page
while current_page < self.config.backfill_depth + self.config.backfill_start_page:
if not self.db_conn.records_loaded:
continue
self.backfill_progress = current_page
self.insert_latest_images(page=current_page, backfill=True)
current_page += 1
time.sleep(self.delay_between_requests)
if self.config.backfill_start_page != original_start_page:
print('Backfill Start Page Changed In Config')
break
self.backfill_progress = 'Completed'
break
else:
self.backfill_progress = 'Disabled'
time.sleep(5)
def _generate_img(self, url=None):
"""
Generate the image files provided from Imgur. We pass the data straight from the request into PIL.Image
"""
img = None
if not url:
return None
try:
response = request.urlopen(url)
img = Image.open(BytesIO(response.read()))
except (HTTPError, ConnectionError, OSError) as e:
msg = 'Error Generating Image File: \n Error Message: {}'.format(e)
self._output_error(msg)
return None
return img if img else None
def generate_latest_images(self, section='user', sort='time', page=0):
self._adjust_rate_limit_timing()
items = []
try:
temp = self.imgur_client.gallery(section=section, sort=sort, page=page, show_viral=False)
if temp:
items = [i for i in temp if not i.is_album and not self.check_post_title(title=i.title)]
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Error Getting Gallery: {}'.format(e)
self._output_error(msg)
return items
def insert_latest_images(self, section='user', sort='time', page=0, backfill=False):
"""
Pull all current images from user sub, get the hashes and insert into database.
"""
# Don't start inserts until all records are loaded
if not self.db_conn.records_loaded:
return
items = self.generate_latest_images(section=section, sort=sort, page=page)
if not items:
return
# Don't add again if we have already done this image ID
for item in items:
if item.id in self.hash_processing.processed_ids:
continue
img = self._generate_img(url=item.link)
if img:
image_hash = self.hash_processing.generate_hash(img)
if image_hash:
record = {
'image_id': item.id,
'url': item.link,
'gallery_url': 'https://imgur.com/gallery/{}'.format(item.id),
'user': item.account_url,
'submitted': item.datetime,
'hash16': image_hash['hash16'],
'hash64': image_hash['hash64'],
'hash256': image_hash['hash256']
}
self.hash_processing.processed_ids.append(item.id)
self.hash_processing.records.append(record)
# If this is called from back filling don't add hash to be checked
if not backfill:
self.hash_processing.hash_queue.append(record)
print('Processing {}'.format(item.link))
else:
print('Backfill Insert {}'.format(item.link))
self.db_conn.add_entry(record)
def downvote_repost(self, image_id):
"""
Downvote the provided Image ID
"""
try:
self.imgur_client.gallery_item_vote(image_id, vote="down")
except ImgurClientError as e:
self.failed_downvotes.append(image_id)
msg = 'Error Voting: {}'.format(e)
self._output_error(msg)
def comment_repost(self, image_id=None, values=None):
"""
Leave a comment on the detected repost.
:param image_id: ID of image to leave comment on.
:param values: Values to be inserted into the message template
:return:
"""
self._output_info('Leaving Comment On {}'.format(image_id))
message = self.build_comment_message(values=values)
if not message:
return
try:
self.imgur_client.gallery_comment(image_id, message)
except (ImgurClientError, ImgurClientRateLimitError) as e:
self.failed_comments.append({'image_id': image_id, 'values': values})
msg = 'Error Posting Commment: {}'.format(e)
self._output_error(msg)
def build_comment_message(self, values=None):
if not values:
return None
# Build up replacement dict
out_dict = {
'count': len(values),
'g_url': values[0]['gallery_url'],
'd_url': values[0]['url'],
'submitted_epoch': values[0]['submitted'],
'submitted_human': time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime(values[0]['submitted'])),
'user': values[0]['user'],
}
try:
final_message = self.config.comment_template.format(**out_dict)
print('Final Message: ' + final_message)
if len(final_message) > 140:
self.logger.warning('Message Length Is Over 140 Chars. Will Be Trimmed')
return final_message
except KeyError as e:
msg = 'Error Generating Message: {}'.format(e)
self._output_error(msg)
return None
def flush_failed_votes_and_comments(self):
"""
If there have been any failed votes or comments (due to imgur server overload) try to redo them
"""
if self.failed_downvotes:
for image_id in self.failed_downvotes:
try:
self.imgur_client.gallery_item_vote(image_id, vote="down")
self.failed_downvotes.remove(image_id)
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Failed To Retry Downvote On Image {}. \nError: {}'.format(image_id, e)
self._output_error(msg)
if self.failed_comments:
for failed in self.failed_comments:
try:
message = self.build_comment_message(values=failed['values'])
self.imgur_client.gallery_comment(failed['image_id'], message)
self.failed_comments.remove(failed['image_id'])
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Failed To Retry Comment On Image {}. \nError: {}'.format(failed['image_id'], e)
self._output_error(msg)
def _repost_processing_thread(self):
"""
Runs in background monitor the queue for detected reposts
:return:
"""
while True:
if len(self.hash_processing.repost_queue) > 0:
current_repost = self.hash_processing.repost_queue.pop(0)
image_id = current_repost[0]['image_id']
sorted_reposts = sorted(current_repost[0]['older_images'], key=itemgetter('submitted'))
if self.config.leave_downvote:
self.downvote_repost(image_id)
if self.config.leave_comment:
self.comment_repost(image_id, values=sorted_reposts)
self.detected_reposts += 1
if self.config.log_reposts:
with open('repost.log', 'a+') as f:
f.write('Repost Image: https://imgur.com/gallery/{}\n'.format(image_id))
f.write('Matching Images:\n')
for r in sorted_reposts:
f.write(r['gallery_url'] + '\n')
def _adjust_rate_limit_timing(self):
"""
Adjust the timing used between request to spread all requests over allowed rate limit
"""
# API Fails To Return This At Times
if not self.imgur_client.credits['ClientRemaining']:
return
remaining_credits_before = int(self.imgur_client.credits['ClientRemaining'])
self.imgur_client.credits = self.imgur_client.get_credits() # Refresh the credit data
# Imgur API sometimes returns 12500 credits remaining in error. If this happens don't update request delay.
# Otherwise the delay will drop to the minimum set in the config and can cause premature credit exhaustion
if int(self.imgur_client.credits['ClientRemaining']) - remaining_credits_before > 100:
"""
print('Imgur API Returned Wrong Remaining Credits. Keeping Last Request Delay Time')
print('API Credits: ' + str(self.imgur_client.credits['ClientRemaining']))
print('Last Credits: ' + str(remaining_credits_before))
"""
return
remaining_credits = self.imgur_client.credits['ClientRemaining']
reset_time = self.imgur_client.credits['UserReset'] + 240 # Add a 4 minute buffer so we don't cut it so close
remaining_seconds = reset_time - round(time.time())
seconds_per_credit = round(remaining_seconds / remaining_credits) # TODO Getting division by zero sometimes
if seconds_per_credit < self.config.min_time_between_requests:
self.delay_between_requests = self.config.min_time_between_requests
else:
self.delay_between_requests = seconds_per_credit
def check_post_title(self, title=None):
"""
Checks the post title for values that we will use to skip over it
This allows us not to flag MRW posts and others as reposts
:return:
"""
if not title:
return None
return [v for v in self.config.title_check_values if v in title.lower()]
def print_current_settings(self):
print('Current Settings')
print('[+] Leave Comments: {}'.format('Enabled' if self.config.leave_comment else 'Disabled'))
print('[+] Leave Downvote: {} '.format('Enabled' if self.config.leave_downvote else 'Disabled'))
print('[+] Backfill: {} '.format('Enabled' if self.config.backfill else 'Disabled'))
print('[+] Backfill Depth: {} '.format(self.config.backfill_depth if self.config.backfill else 'Disabled'))
print('[+] Process Pool Size: {} '.format(self.config.hash_proc_limit))
print('[+] Hash Size: {} bit'.format(self.config.hash_size))
print('[+] Hamming Distance: {}{}'.format(self.config.hamming_cutoff, '\n'))
def print_current_stats(self):
print('Current Stats')
print('[+] Total Images In Database: {}'.format(str(len(self.hash_processing.processed_ids))))
print('[+] Total Hashes Waiting In Pool: {}'.format(str(self.hash_processing.total_in_queue)))
print('[+] Total Hashes In Hash Queue: {}'.format(str(len(self.hash_processing.hash_queue))))
print('[+] Process Pool Status: {}'.format(self.hash_processing.pool_status))
print('[+] Total Reposts Found: {}'.format(str(self.detected_reposts)))
print('[+] Backfill Progress: {}\n'.format('Page ' + str(self.backfill_progress) if self.config.backfill else 'Disabled'))
def print_api_stats(self):
print('API Settings')
print('[+] Remaining Credits: {}'.format(self.imgur_client.credits['ClientRemaining']))
if self.imgur_client.credits['UserReset']:
print('[+] Time Until Credit Reset: {} Minutes'.format(round((int(self.imgur_client.credits['UserReset']) - time.time()) / 60)))
# Make it clear we are overriding the default delay to meet credit refill window
if self.delay_between_requests == self.config.min_time_between_requests:
request_delay = str(self.delay_between_requests) + ' Seconds'
else:
request_delay = str(self.delay_between_requests) + ' Seconds (Overridden By Rate Limit)'
print('[+] Delay Between Requests: {} \n'.format(request_delay))
def run(self):
last_run = round(time.time())
while True:
os.system('cls')
self.print_current_stats()
self.print_current_settings()
self.print_api_stats()
if round(time.time()) - last_run > self.delay_between_requests:
self.insert_latest_images()
self.flush_failed_votes_and_comments()
last_run = round(time.time())
self._check_thread_status()
time.sleep(2)
from imgurpython import ImgurClient
client_id = '51a86b471a85443'
client_secret = '442df4f265e082d2c168ad2ea7da3607b12b37e8'
target_file = str(sys.argv[1])
client = ImgurClient(client_id, client_secret)
target = open(target_file, 'w')
# Example request
for i in range(10):
items = client.gallery(section='hot', sort='viral', page=i, window='day', show_viral=True)
for item in items:
if ".jpg" in item.link:
target.write(str(item.link))
target.write("\n")
print item.link
target.close()
from imgurpython import ImgurClient
PATH = "imgur1.txt"
client_id = 'b68fbc6850008f8'
client_secret = 'b83cc295315d15e167a61d3a8a572ee5bf42d731'
imgur_client = ImgurClient(client_id, client_secret)
imgur_gallery = imgur_client.gallery()
# CMU 15-112
def writeFile(filename, contents, mode="wt"):
with open(filename, mode) as fout:
fout.write(contents)
def main():
result = ""
items = imgur_gallery
extensionIndex = -4
# get random images
for item in items:
# image link
if item.link[extensionIndex] == ".":
result += item.link + "\n"
writeFile(PATH, result)
if __name__ == "__main__":
main()
from imgurpython import ImgurClient
import json
client_id = '##############'
client_secret = '#######################'
client = ImgurClient(client_id, client_secret)
def convert_list_to_string(mlist):
converted_list = str(mlist).strip('[]')
converted_list = converted_list.replace('\'', '')
return converted_list
image_list = []
# Example request
for i in range(0, 50):
print("PAGE: " + str(i))
items = client.gallery(page=0)
for item in items:
image_list.append(item.link)
json_out = {"Images": convert_list_to_string(image_list)}
with open('images.json', 'w') as outfile:
json.dump(json_out, outfile)
