class ImageRetriever: def __init__(self): self.logger = getLogger(__name__) self.client = ImgurClient(getenv('IMGUR_CLIENT_ID'), getenv('IMGUR_CLIENT_SECRET')) def retrieve_images(self, no_pages=10): images = filter( lambda item: isinstance(item, GalleryImage) and item.animated, reduce(add, (self.client.gallery(page=i) for i in range(no_pages)))) pipe = app.db.pipeline() pipe.delete('images') for image in images: print("Storing {}".format(image.id)) pipe.hsetnx('images', image.id, dumps(image)) pipe.execute() def get_images(self): return list(map(loads, app.db.hvals('images'))) def get_image(self, image_id): try: return loads(app.db.hget('images', image_id)) except TypeError: raise KeyError()
async def imgur_meme(message, bot): clientId = 'b49ec8d9045f4ff' clientSecret = 'ff33b374c6c0faf7c591020cf621afdd963d1ea4' try: imager = ImgurClient(clientId, clientSecret) randPage = random.randint(0, 10) await bot.send_message(message.channel, "Fetching a spicy one...") items = imager.gallery(page=randPage) while True: randImage = random.randint(0, len(items)) try: if str(items[randImage].images[0]['link'])[-4:] == '.gif': continue await bot.send_message(message.channel, "```" + items[randImage].title + "```") await bot.send_message(message.channel, items[randImage].images[0]['link']) return except AttributeError: if str(items[randImage].link)[-4:] == '.gif': continue await bot.send_message(message.channel, "```" + items[randImage].title + "```") await bot.send_message(message.channel, items[randImage].link) return except JSONDecodeError: continue except ImgurClientError: continue except CancelledError as e: await bot.send_message(message.channel, "Imgur Cancelled Error: let Skittlely know") print(e) except TimeoutError as e: await bot.send_message(message.channel, "Imgur Timeout Error: let Skittlely know") print(e)
class ImageRetriever: def __init__(self): self.logger = getLogger(__name__) self.client = ImgurClient( getenv('IMGUR_CLIENT_ID'), getenv('IMGUR_CLIENT_SECRET')) def retrieve_images(self, no_pages=10): images = filter( lambda item: isinstance(item, GalleryImage) and item.animated, reduce(add, (self.client.gallery(page=i) for i in range(no_pages))) ) pipe = app.db.pipeline() pipe.delete('images') for image in images: print("Storing {}".format(image.id)) pipe.hsetnx('images', image.id, dumps(image)) pipe.execute() def get_images(self): return list(map(loads, app.db.hvals('images'))) def get_image(self, image_id): try: return loads(app.db.hget('images', image_id)) except TypeError: raise KeyError()
def Random_Pic(): client = ImgurClient(client_id, client_secret) items = client.gallery() num = random.randint(0, len(items)) return items[num].link
async def imgur(message, client): """ Imgur stuff :param message: :param client: :return: """ im_client_id = "c8093a713f38daa" im_client_secret = "69a6863b617b86d562c6d45e45d9daf9134c283b" im_client = ImgurClient(im_client_id, im_client_secret) params = message.content.split() paramslength = int(len(params)) try: items = im_client.gallery(section='hot', sort='viral', page=0, window='day', show_viral=True) get = int(params[2]) for num in range(0, get): for item in items: await client.send_message(message.channel, item.link) except ImgurClientError as e: print(e.error_message) print(e.status_code)
def get_imgur_links(client_id, client_secret): try: client = ImgurClient(client_id, client_secret) except ImgurClientError: get_imgur_links(client_id, client_secret) items = client.gallery() return [x.link for x in items if x is not None]
def random_image(): load_dotenv() client_id = os.getenv('IMGUR_TOKEN') client_secret = os.getenv('IMGUR_SECRET') client = ImgurClient(client_id, client_secret) items = client.gallery() item = random.choice(items) return item.link
def BikiniBottomTwitter(): load_dotenv() client_id = os.getenv('IMGUR_TOKEN') client_secret = os.getenv('IMGUR_SECRET') client = ImgurClient(client_id, client_secret) items = client.gallery(section='r', sort='BikiniBottomTwitter', show_viral=False) item = random.choice(items) return item.link
def get_meme(): try: client = ImgurClient(ImgurClientID, ImgurClientSecret) except BaseException as e: print('[EROR] Error while authenticating with Imgur:', str(e)) return items = client.gallery() list = [] for item in items: list.append(item.link) return random.choice(list)
def getImageLinks(pagenum=1, max_links=50): #initialize client client = ImgurClient(imgur_client_id, imgur_client_secret) #Get items items = client.gallery(section=CURRENT_SUBJECT, sort='time', page=pagenum, window='day', show_viral=False) #Get links links = [item.link for item in items] #Get image titles titles = [item.title for item in items] #Return list of links return links[:max_links], titles[:max_links]
def imgur(bot, chat_id): client_id = config.IMGUR_ID client_secret = config.IMGUR_SECRET client = ImgurClient(client_id, client_secret) items = [] for i in range(0, 3): items += client.gallery(section='hot', sort='viral', page=i, window='day') item = items[random.randrange(0, len(items))] bot.send_photo(chat_id, photo=item.link, caption=item.title, reply_markup=markup)
def getImageLinks(pagenum=1, max_links=50): #initialize client client = ImgurClient(imgur_client_id, imgur_client_secret) #Get items items = client.gallery(section=CURRENT_SUBJECT, sort='time', page=pagenum, window='day', show_viral=False) #Get links links = [item.link for item in items] #Get image titles titles = [item.title for item in items] #Return list of links return links[:max_links], titles[:max_links]
def get_imgur_image(text): client_id = os.environ.get('IMGUR_CLIENT_ID') client_secret = os.environ.get('IMGUR_CLIENT_SECRET') if client_id and client_secret: client = ImgurClient(client_id, client_secret) tag = tag_from_text(text) if tag: try: gallery = client.gallery_tag(tag) gallery = gallery.items except ImgurClientError: return 'Tag {} not found'.format(tag) else: gallery = client.gallery() return random_link_from_gallery_list(gallery)
def imgur(request, section='user', sort='rising', page=0): if request.method == "POST": form = GrabberForm(request.POST) if form.is_valid(): post = form.save(commit=False) post.time_added = datetime.now() post.downloaded = False post.save() return redirect('post_detail', pk=post.pk) else: form = GrabberForm() page = int(page) client_id = getattr(settings, 'IMGUR_CLIENT_ID', '') client_secret = getattr(settings, 'IMGUR_CLIENT_TOKEN', '') client = ImgurClient(client_id, client_secret) current_limits = client.credits items = client.gallery(section=section, sort=sort, page=page, show_viral=True) album_images = {} form = GrabberForm() # images = client.get_album_images(items[1].id) # Grab all the images in album for item in items: if item.is_album is True: id = item.id images = client.get_album_images(item.id) album_images[id] = images # album_images[id] = "test" return render( request, 'webapp/dashboard.html', { 'items': items, 'current_limits': current_limits, 'album_images': album_images, 'section': section, 'sort': sort, 'page': page, 'form': form, })
def index(): client = ImgurClient(app.config['IMGUR_ID'], app.config['IMGUR_SECRET']) try: albums = client.gallery(sort='top', window='week', page=0) except ConnectionError: logger.error('Connection error') return images = list(islice((a for a in albums if matches_reqs(a)), 0, 10)) for img in images: if img.account_id: author = User.query.get(img.account_id) if not author: author = User(id=img.account_id, nickname=img.account_url) db.session.add(author) if not Album.query.get(img.id): a = Album(id=img.id, link=img.link, author=author, title=img.title) db.session.add(a) db.session.commit() return render_template( "index.html", images=images)
async def imgurCommand(message): imgr = ImgurClient(IMGUR_ID, IMGUR_SECRET) cmd = message.content.split(" ", 1) if len(cmd) <= 1: items = imgr.gallery(section='user', sort='time', page=1, window='day', show_viral=False) else: items = imgr.gallery_search(cmd[1], advanced=None, sort='time', window='all', page=0) if len(items) < 1: await message.channel.send("No reults") i = random.randint( 0, len(items) - 1) # Added -1 to prevent list index out of range exceptions await message.channel.send(items[i].link)
def send_email(): genericLogger.info('Started send_email task') client = ImgurClient(settings.IMGUR['CLIENT_ID'], settings.IMGUR['CLIENT_SECRET']) try: gallery = client.gallery(sort='top', window='week', page=0) except ConnectionError: genericLogger.error('Connection error') return images = islice((item for item in gallery if matches_reqs(item)), 0, 10) for s in Subscriber.objects.all(): genericLogger.info('Started user {}'.format(s.email)) email_content = generate_email(settings.HOST, s.name, s.email, images) try: send_mail(subject="Enjoy Most Popular ImgUr Images for the Last Week", message="", from_email='*****@*****.**', recipient_list=[s.email], html_message=email_content) genericLogger.info('Sent email to {}'.format(s.email)) except SMTPException as e: # invalid email ? genericLogger.error(e)
from imgurpython import ImgurClient client_id = 'YOUR CLIENT ID' client_secret = 'YOUR CLIENT SECRET' client = ImgurClient(client_id, client_secret) # Example request items = client.gallery() for item in items: print(item.link+",") # Go th https://imgur.com/removalrequest to delete the image you upload
from imgurpython import ImgurClient import random client_id = "" client_secret = "" client = ImgurClient(client_id, client_secret) links = [] items = client.gallery(section="/r/wallpapers", sort="top", window="all") #gets image links from wallpaper section for item in items: links.append(item.link) #put all links on an array def returnID(): # get a random link from links array and returns it x = random.randrange(0, len(links)) return links[x]
def sms_reply(): # Gets the SMS sent to the number body = request.values.get('Body', "") check = command_check() resp = MessagingResponse() needshelp = 0 # !echo if check == 1: body = body[6:] # Checks for parameters if body == None: resp.message("!echo requires additional input.") else: resp.message(body) # Echoes return str(resp) # !wiki elif check == 2: body = body[6:] # Checks for parameters if body == None: resp.message("!wiki requires additonal input.") return str(resp) # Search for article try: search = wikipedia.search(body, results=1, suggestion=True) except wikipedia.exceptions.PageError as e: resp.message("Error generating Wikipedia page.") return str(resp) # Removes disambiguation error try: article = wikipedia.page(title=search[0]) except wikipedia.exceptions.DisambiguationError as e: article = wikipedia.page(title=e.options[0]) # Returns article info title = article.title summary = wikipedia.summary(title, sentences=3) url = article.url # Submits article info wiki_stuff = title + "\n\n" + summary + "\n\n" + "Find out more at:" + url resp.message(wiki_stuff) return str(resp) # !stock elif check == 3: body = body[7:] # Checks for parameters if body == None: resp.message("!stock requires additonal input.") body = body.upper() stocklink = "https://finance.google.com/finance?q=" + body + "&output=json" rsp = requests.get(stocklink) if rsp.status_code in (200,): fin_data = json.loads(rsp.content[6:-2].decode('unicode_escape')) openprice = ('Opening Price: ${}'.format(fin_data['op'])) stockname = ('Stock Name: {}'.format(fin_data['name'])) stockinfo = stockname + "\n" + "Symbol: " + body + "\n" + openprice if format(fin_data['op']) == "": stockinfo = "Invalid stock symbol \n\nSymbols: Facebook(FB), Apple(AAPL), Alphabet(GOOG), Yahoo(YHOO), Amazon(AMZN), Coca-Cola(KO), Walmart(WMT), Microsoft(MSFT)\n\nMore at: https://finance.yahoo.com/" resp.message(stockinfo) return str(resp) # !help elif check == 0: return help() # !rand elif check == 4: body = body[6:] # If no parameter is applied if body == None: items = ImgurClient.gallery() randnum = random(0, len(items)-1) img = get_image(items[randnum].image_id) return img else: items = ImgurClient.gallery_search(q=body, advanced=None, sort='time', window='all', page=0) randnum = random(0, len(items)-1) img = get_image(items[randnum].image_id) return img # 5 Failed Commands = !help else: needshelp = needshelp + 1 if needshelp >= 5: needshelp = 0 help() return
from imgurpython import ImgurClient from client_data import get_id, get_secret client_id = get_id() client_secret = get_secret() client = ImgurClient(client_id, client_secret) # Example request items = client.gallery() for item in items: #print(item.link) pass def test_test(number): return number + 10
class ImgUr(object): """ @Params: {string} - Download Directory @Returns: None """ def __init__(self, dd=None): client_id = '50049400837.73078400742' client_secret = '5a730c87aeea8c2daad196ef3ff2c6da' if not dd: self.download_dir = 'downloads' else: self.download_dir = dd self.setup_download_dir(self.download_dir) self.data = [] self.client = ImgurClient(client_id, client_secret) self.http = urllib3.PoolManager() """ @Params: {int} - Minimum number of images to download @Description: This method uses the ImgurClient to download pages of images from imgur and places them into a list. @Returns: {list} - List of urls to images """ def get_links(self, min=10): page = 0 while len(self.data) <= min: items = self.client.gallery(page=page, window='day', show_viral=True) for item in items: name, ext = os.path.splitext(item.link) if ext == '.jpg' or ext == '.png': self.data.append(item.link) page += 1 return self.data """ @Params: {string} - Url of image to download @Description: This method receives a url and downloads it using the urllib3 library. Saves it to a directory determined by the class constructor. @Returns: None """ def download_link(self, link): path = self.download_dir + "/" + os.path.basename(link) r = self.http.request('GET', link, preload_content=False) with open(path, 'wb') as out: while True: data = r.read() if not data: break out.write(data) r.release_conn() """ @Params: {string} - Download directory name @Description: This will check to see if the param is a directory and if not it will create it. @Returns: None """ def setup_download_dir(self, d): self.download_dir = d if not os.path.isdir(self.download_dir): os.makedirs(self.download_dir)
png_count = 0 gif_count = 0 def countImage( type ): global jpeg_count global png_count global gif_count if(type == "image/jpeg"): jpeg_count += 1 elif(type == "image/png"): png_count += 1 elif(type == "image/gif"): gif_count += 1 # Crawling Requests items = client.gallery(page=0) for item in items: print(item.link) if(not item.is_album): countImage(item.type) else: for image in client.get_album_images(item.id): countImage(image.type) total_count = jpeg_count + png_count + gif_count print("JPEG:" + str(jpeg_count) + " " + str(round(100 * (float(jpeg_count) / float(total_count)), 2)) + "%") print("PNG:" + str(png_count) + " " + str(round(100 * (float(png_count) / float(total_count)), 2)) + "%") print("GIF:" + str(gif_count) + " " + str(round(100 * (float(gif_count) / float(total_count)), 2)) + "%") print("Total:" + str(total_count))
client = ImgurClient(client_id, client_secret, access_token, refresh_token) # url = "https://api.imgur.com/3/gallery/top/time/day/2?showViral=false&mature=false&album_previews=false" # headers = {'Authorization': 'Client-ID 9b315a3d4a73278'} # response = requests.request("GET", url, headers=headers) # print(response) # res = requests.get('https://api.imgur.com/3/gallery/section/sort/window/page?showViral=false&mature=false&album_previews=false', params={'section': 'top', 'sort': 'viral', 'window': 'week', 'page': 3}) # print(res) # Extracts the items (images) on the front page of Imgur: # PROBLEM = I'm able to get the links, but not make it appear on the page with a Jinja template. items = client.gallery(section='hot', sort='viral', page=2, window='week', show_viral=False) for item in items: print(item.link) # print(item.title) # print(items) # create jinja for loop to iterate through each one to make a card # Find the image on the front page that has the highest number of views # items = client.gallery() # max_item = None # max_views = 0 # for item in items: # if item.views > max_views: # max_item = item # max_views = item.views # print(max_item.title)
class ImgurSpider(Spider): name = 'imgur' imgur = None def setup_filter(self, sf): sf.add_com('post', info='A post.') sf.add_com('comment', info='A comment in reply to a post.') sf_post = sf.get('post') sf_post.add('title', info='The title of the post.') sf_post.add('description', info='The description of the post.') sf_post.add('author', info='The user who submitted the post.') sf_post.add('points', info='The points of the post.') sf_post.add('score', info='The score of the post.') sf_post.add('ups', info='The number upvotes the post get.') sf_post.add('downs', info='The number of downvotes the post get.') sf_post.add('views', info='The view count of the post.') sf_post.add('tags', info='The tags labelled on the post.') sf_post.add('nsfw', info='States whether or not the post is NSFW.') sf_comment = sf.get('comment') sf_comment.add('body', info='The body of the comment.') sf_comment.add('author', info='The user who submitted the comment.') sf_comment.add('points', info='The points of the comment.') sf_comment.add('ups', info='The upvotes the comment get.') sf_comment.add('downs', info='The downvotes the comment get.') sf.add_var( 'sections', type=list, default=['hot', 'new'], info="Determine which sections posts are going to be "\ "retrieved from. The 'hot' section is labelled as "\ "'most viral' "\ "in imgur.com, and the 'new' section is labelled as "\ "'user submitted'" ) sf.add_var( 'time', type=str, choice=['day', 'week', 'month',\ 'year', 'all'], default='day', info="The time frame of when the post was submitted. Note that "\ "this only applies to a 'hot' section with 'top' sorting. " ) sf.add_var( 'sort', type=str, choice=['popularity', 'top', 'newest'], default='popularity', info="Choose the order in how the posts are to be scanned. "\ "Note that the 'top' choice can be configured further "\ "with the @time attribute." ) sf.add_var('skip_comments', type=bool, default=False, info='Skip scanning over comments for every scanned post.') sf.add_var( 'page_limit', type=int, default=999999, info="The limit on how many page will the spider scan before it "\ "stops. The default is -1 (no limit)." ) def start(self, sf): client_id = '93814a7ab6dccf6' client_secret = 'c9ed8ffe67e553f10f8d587f5d335ae68264fd92' self.imgur = ImgurClient(client_id, client_secret) for item in self.generator(sf): yield item def generator(self, sf): """ An item list generator containing the items in a page. """ sf_sections = sf.ret('sections') sf_time = sf.ret('time') sf_sort = sf.ret('sort') p = 0 for section in sf_sections: while True: if p >= sf.ret('page_limit'): return page = self.imgur.gallery(section=section, sort=sf_sort, page=p, window=sf.ret('time')) p += 1 for post in self.generate_post(sf, page): yield post def generate_post(self, sf, page): for post in page: sf.get('post').set_attr_values( author=post.account_url, title=post.title, description=post.description, views=post.views, points=int(post.points), score=int(post.score), ups=int(post.ups), downs=int(post.downs), tags=[tag['name'] for tag in post.tags], nsfw=bool(post.nsfw)) if sf.get('post').should_scrape(): yield ComponentLoader( 'post', { 'post_id': post.id, 'author': post.account_url, 'title': post.title, 'description': post.description, 'link': post.link, 'topic': post.topic, 'points': post.points, 'score': post.score, 'ups': post.ups, 'downs': post.downs, 'comment_count': post.comment_count, 'tags': ','.join([tag['name'] for tag in post.tags]), 'nsfw': post.nsfw }) if sf.ret('skip_comments'): continue for comment in self.generate_comment(sf, post.id): yield comment def generate_comment(self, sf, post_id): for comment in self.imgur.gallery_item_comments(post_id): sf.get('comment').set_attr_values(body=comment.comment, author=comment.author, points=comment.points, ups=comment.ups, downs=comment.downs) if sf.get('comment').should_scrape(): yield ComponentLoader( 'comment', { 'comment_id': comment.id, 'parent_id': comment.parent_id, 'body': comment.comment, 'author': comment.author, 'datetime': comment.datetime, 'deleted': comment.deleted, 'points': comment.points, 'vote': comment.vote, 'downs': comment.downs, 'ups': comment.ups, })
class Imgur: """The most awesome images on the internet! This plugin allows basic searching on Imgur including subreddits. Warning: Searching on subreddits cannot be easily moderated, therefore it is extremely easy for a user to post images from an nsfw subreddit to whatever channel the bot is enabled in if this plugin is enabled without modification. The subreddit command can be disabled by changing 'enabled=True' to 'enabled=False' in the plugin's main file: 'plugins/imgur.py' on line 53. """ def __init__(self, bot): self.bot = bot self.client = ImgurClient(config["client_id"], config["client_secret"]) @c.group(pass_context=True) async def imgur(self, ctx): """Search on Imgur!""" if ctx.invoked_subcommand is None: await self.bot.say("Zoinks! You've taken a wrong turn! Try `help imgur`.") # Helper function to actually get/post the images async def post_image(self, request, query=None): case = { "subreddit": lambda: self.client.subreddit_gallery(query), "search" : lambda: self.client.gallery_search(query), "random" : lambda: self.client.gallery_random(), "top" : lambda: self.client.gallery("top"), "hot" : lambda: self.client.gallery("hot"), "rehost" : lambda: self.client.upload_from_url(query), } function = case.get(request, None) image = self.bot.loop.run_in_executor(None, function) while True: await asyncio.sleep(0.25) if image.done(): image = image.result() break if request == "rehost": await self.bot.say(image.get("link", None)) else: await self.bot.say(random.choice(image).link) @imgur.command(name="sub", aliases=["subreddit", "reddit", "r/"], enabled=True) async def imgur_subreddit(self, subreddit: str): """Get an image from a subreddit.""" await self.post_image("subreddit", subreddit) @imgur.command(name="search") async def imgur_search(self, *, query: str): """Search Imgur for (almost) anything.""" await self.post_image("search", query) @imgur.command(name="random") async def imgur_random(self): """One free random image.""" await self.post_image("random") @imgur.command(name="viral") async def imgur_viral(self, section: str = "top"): """Get one of the most viral images of the day. Section may be either 'top' or 'hot' and will get an image based on that criteria.""" section = section.lower() if section != "top": section = "hot" await self.post_image(section) @imgur.command(name="rehost") async def imgur_rehost(self, url: str): """Rehost an image from any link to Imgur.""" await self.post_image("rehost", url)
import discord import random import asyncio from discord.ext import commands from imgurpython import ImgurClient import os #clients or defining variables# client_id = os.environ['client_id'] client_secret = os.environ['client_secret'] grab = ImgurClient( client_id, client_secret ) #had to give it a diffrent call to function from client as Discords prefix utilises "client" call to function# items = grab.gallery() class SFWCog: def __init__(self, client): self.client = client async def on_ready(self): print('SFWCog is Prepared') #Imgur commands# #memes and gifs# @commands.command( pass_context=True, description= "used for testing JSON file; can be used as .code_test **subtypename**"
import json import os from random import shuffle with open('twitter_keys.txt') as json_file: keys = json.load(json_file) while True: try: #Imgur client_id = keys['imgur'][0]['client_id'] client_secret = keys['imgur'][0]['client_secret'] client = ImgurClient(client_id, client_secret) items=client.gallery(window='hour') shuffle(items) #tags_list = [] keyWords=['politics', 'vote', 'capitalism','trump', 'current events', 'republican', 'hatchact', 'election', 'fucktrump'] keyWords2 = ['funny', 'memes'] index = 0 size = len(items) while index < size:
import datetime as dt import requests import csv import sys extract_date = dt.datetime.now() #import config file and read in the values config = configparser.ConfigParser() config.read('auth.ini') client_id = config.get('credentials', 'client_id') client_secret = config.get('credentials', 'client_secret') #open connection to imgur and retrieve all gallery images client = ImgurClient(client_id, client_secret) images = client.gallery() image_info = [] image_tags = [] print('Processing.', end='') #go through the gallery images and record the necessary fields; images and albums are treated differently for image in images: print('.', end='') sys.stdout.flush() if image.is_album == True: #record fields for this album image_info.append([ image.id, image.title, 'Album', image.views, image.ups, image.downs, image.points, image.comment_count,
class ImgurRepostBot(): def __init__(self): os.system('cls') self.failed_downvotes = [] # Store failed downvotes for later processing self.failed_comments = [] # Store failed comments for later processing self.delay_between_requests = 5 # Changed on the fly depending on remaining credits and time until reset self.thread_lock = threading.Lock() self.logger = None self.detected_reposts = 0 self.config = ConfigManager() self._setup_logging() self.imgur_client = ImgurClient(self.config.api_details['client_id'], self.config.api_details['client_secret'], self.config.api_details['access_token'], self.config.api_details['refresh_token']) self.db_conn = ImgurRepostDB(self.config) self.backfill_progress = 1 if self.config.backfill else 'Disabled' records, processed_ids = self.db_conn.build_existing_ids() if self.config.backfill: threading.Thread(target=self._backfill_database, name='Backfill').start() self.hash_processing = HashProcessing(self.config, processed_ids, records) threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start() def _check_thread_status(self): """ Check status of critical threads. If they are found dead start them back up :return: """ thread_names = ['configmonitor', 'backfill', 'repostprocessing'] for thrd in threading.enumerate(): if thrd.name.lower() in thread_names: thread_names.remove(thrd.name.lower()) for i in thread_names: if i == 'configmonitor': msg = 'Config Monitor Thread Crashed' self._output_error(msg) self.config = ConfigManager() continue if i == 'backfill' and self.config.backfill: msg = 'Backfill Thread Crashed' self._output_error(msg) threading.Thread(target=self._backfill_database, name='Backfill').start() continue if i == 'repostprocessing': msg = 'Repost Processing Thread Crashed' self._output_error(msg) threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start() continue def _setup_logging(self): if self.config.logging: self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.ERROR) formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') fhandle = logging.FileHandler('botlog.log') fhandle.setFormatter(formatter) self.logger.addHandler(fhandle) def _output_error(self, msg, output=True): """ convenience method to log and/or print an error :param msg: Message to output/log :param output: Print error to console :return: """ if output: print(msg) if self.config.logging: self.logger.error(msg) def _output_info(self, msg): print(msg) if self.config.logging: self.logger.info(msg) def _backfill_database(self): """ Backfill the database with older posts. Useful if script hasn't been run in some time" :return: """ while True: if self.config.backfill: original_start_page = self.config.backfill_start_page # So we can detect if it's changed in the config current_page = self.config.backfill_start_page while current_page < self.config.backfill_depth + self.config.backfill_start_page: if not self.db_conn.records_loaded: continue self.backfill_progress = current_page self.insert_latest_images(page=current_page, backfill=True) current_page += 1 time.sleep(self.delay_between_requests) if self.config.backfill_start_page != original_start_page: print('Backfill Start Page Changed In Config') break self.backfill_progress = 'Completed' break else: self.backfill_progress = 'Disabled' time.sleep(5) def _generate_img(self, url=None): """ Generate the image files provided from Imgur. We pass the data straight from the request into PIL.Image """ img = None if not url: return None try: response = request.urlopen(url) img = Image.open(BytesIO(response.read())) except (HTTPError, ConnectionError, OSError) as e: msg = 'Error Generating Image File: \n Error Message: {}'.format(e) self._output_error(msg) return None return img if img else None def generate_latest_images(self, section='user', sort='time', page=0): self._adjust_rate_limit_timing() items = [] try: temp = self.imgur_client.gallery(section=section, sort=sort, page=page, show_viral=False) if temp: items = [i for i in temp if not i.is_album and not self.check_post_title(title=i.title)] except (ImgurClientError, ImgurClientRateLimitError) as e: msg = 'Error Getting Gallery: {}'.format(e) self._output_error(msg) return items def insert_latest_images(self, section='user', sort='time', page=0, backfill=False): """ Pull all current images from user sub, get the hashes and insert into database. """ # Don't start inserts until all records are loaded if not self.db_conn.records_loaded: return items = self.generate_latest_images(section=section, sort=sort, page=page) if not items: return # Don't add again if we have already done this image ID for item in items: if item.id in self.hash_processing.processed_ids: continue img = self._generate_img(url=item.link) if img: image_hash = self.hash_processing.generate_hash(img) if image_hash: record = { 'image_id': item.id, 'url': item.link, 'gallery_url': 'https://imgur.com/gallery/{}'.format(item.id), 'user': item.account_url, 'submitted': item.datetime, 'hash16': image_hash['hash16'], 'hash64': image_hash['hash64'], 'hash256': image_hash['hash256'] } self.hash_processing.processed_ids.append(item.id) self.hash_processing.records.append(record) # If this is called from back filling don't add hash to be checked if not backfill: self.hash_processing.hash_queue.append(record) print('Processing {}'.format(item.link)) else: print('Backfill Insert {}'.format(item.link)) self.db_conn.add_entry(record) def downvote_repost(self, image_id): """ Downvote the provided Image ID """ try: self.imgur_client.gallery_item_vote(image_id, vote="down") except ImgurClientError as e: self.failed_downvotes.append(image_id) msg = 'Error Voting: {}'.format(e) self._output_error(msg) def comment_repost(self, image_id=None, values=None): """ Leave a comment on the detected repost. :param image_id: ID of image to leave comment on. :param values: Values to be inserted into the message template :return: """ self._output_info('Leaving Comment On {}'.format(image_id)) message = self.build_comment_message(values=values) if not message: return try: self.imgur_client.gallery_comment(image_id, message) except (ImgurClientError, ImgurClientRateLimitError) as e: self.failed_comments.append({'image_id': image_id, 'values': values}) msg = 'Error Posting Commment: {}'.format(e) self._output_error(msg) def build_comment_message(self, values=None): if not values: return None # Build up replacement dict out_dict = { 'count': len(values), 'g_url': values[0]['gallery_url'], 'd_url': values[0]['url'], 'submitted_epoch': values[0]['submitted'], 'submitted_human': time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime(values[0]['submitted'])), 'user': values[0]['user'], } try: final_message = self.config.comment_template.format(**out_dict) print('Final Message: ' + final_message) if len(final_message) > 140: self.logger.warning('Message Length Is Over 140 Chars. Will Be Trimmed') return final_message except KeyError as e: msg = 'Error Generating Message: {}'.format(e) self._output_error(msg) return None def flush_failed_votes_and_comments(self): """ If there have been any failed votes or comments (due to imgur server overload) try to redo them """ if self.failed_downvotes: for image_id in self.failed_downvotes: try: self.imgur_client.gallery_item_vote(image_id, vote="down") self.failed_downvotes.remove(image_id) except (ImgurClientError, ImgurClientRateLimitError) as e: msg = 'Failed To Retry Downvote On Image {}. \nError: {}'.format(image_id, e) self._output_error(msg) if self.failed_comments: for failed in self.failed_comments: try: message = self.build_comment_message(values=failed['values']) self.imgur_client.gallery_comment(failed['image_id'], message) self.failed_comments.remove(failed['image_id']) except (ImgurClientError, ImgurClientRateLimitError) as e: msg = 'Failed To Retry Comment On Image {}. \nError: {}'.format(failed['image_id'], e) self._output_error(msg) def _repost_processing_thread(self): """ Runs in background monitor the queue for detected reposts :return: """ while True: if len(self.hash_processing.repost_queue) > 0: current_repost = self.hash_processing.repost_queue.pop(0) image_id = current_repost[0]['image_id'] sorted_reposts = sorted(current_repost[0]['older_images'], key=itemgetter('submitted')) if self.config.leave_downvote: self.downvote_repost(image_id) if self.config.leave_comment: self.comment_repost(image_id, values=sorted_reposts) self.detected_reposts += 1 if self.config.log_reposts: with open('repost.log', 'a+') as f: f.write('Repost Image: https://imgur.com/gallery/{}\n'.format(image_id)) f.write('Matching Images:\n') for r in sorted_reposts: f.write(r['gallery_url'] + '\n') def _adjust_rate_limit_timing(self): """ Adjust the timing used between request to spread all requests over allowed rate limit """ # API Fails To Return This At Times if not self.imgur_client.credits['ClientRemaining']: return remaining_credits_before = int(self.imgur_client.credits['ClientRemaining']) self.imgur_client.credits = self.imgur_client.get_credits() # Refresh the credit data # Imgur API sometimes returns 12500 credits remaining in error. If this happens don't update request delay. # Otherwise the delay will drop to the minimum set in the config and can cause premature credit exhaustion if int(self.imgur_client.credits['ClientRemaining']) - remaining_credits_before > 100: """ print('Imgur API Returned Wrong Remaining Credits. Keeping Last Request Delay Time') print('API Credits: ' + str(self.imgur_client.credits['ClientRemaining'])) print('Last Credits: ' + str(remaining_credits_before)) """ return remaining_credits = self.imgur_client.credits['ClientRemaining'] reset_time = self.imgur_client.credits['UserReset'] + 240 # Add a 4 minute buffer so we don't cut it so close remaining_seconds = reset_time - round(time.time()) seconds_per_credit = round(remaining_seconds / remaining_credits) # TODO Getting division by zero sometimes if seconds_per_credit < self.config.min_time_between_requests: self.delay_between_requests = self.config.min_time_between_requests else: self.delay_between_requests = seconds_per_credit def check_post_title(self, title=None): """ Checks the post title for values that we will use to skip over it This allows us not to flag MRW posts and others as reposts :return: """ if not title: return None return [v for v in self.config.title_check_values if v in title.lower()] def print_current_settings(self): print('Current Settings') print('[+] Leave Comments: {}'.format('Enabled' if self.config.leave_comment else 'Disabled')) print('[+] Leave Downvote: {} '.format('Enabled' if self.config.leave_downvote else 'Disabled')) print('[+] Backfill: {} '.format('Enabled' if self.config.backfill else 'Disabled')) print('[+] Backfill Depth: {} '.format(self.config.backfill_depth if self.config.backfill else 'Disabled')) print('[+] Process Pool Size: {} '.format(self.config.hash_proc_limit)) print('[+] Hash Size: {} bit'.format(self.config.hash_size)) print('[+] Hamming Distance: {}{}'.format(self.config.hamming_cutoff, '\n')) def print_current_stats(self): print('Current Stats') print('[+] Total Images In Database: {}'.format(str(len(self.hash_processing.processed_ids)))) print('[+] Total Hashes Waiting In Pool: {}'.format(str(self.hash_processing.total_in_queue))) print('[+] Total Hashes In Hash Queue: {}'.format(str(len(self.hash_processing.hash_queue)))) print('[+] Process Pool Status: {}'.format(self.hash_processing.pool_status)) print('[+] Total Reposts Found: {}'.format(str(self.detected_reposts))) print('[+] Backfill Progress: {}\n'.format('Page ' + str(self.backfill_progress) if self.config.backfill else 'Disabled')) def print_api_stats(self): print('API Settings') print('[+] Remaining Credits: {}'.format(self.imgur_client.credits['ClientRemaining'])) if self.imgur_client.credits['UserReset']: print('[+] Time Until Credit Reset: {} Minutes'.format(round((int(self.imgur_client.credits['UserReset']) - time.time()) / 60))) # Make it clear we are overriding the default delay to meet credit refill window if self.delay_between_requests == self.config.min_time_between_requests: request_delay = str(self.delay_between_requests) + ' Seconds' else: request_delay = str(self.delay_between_requests) + ' Seconds (Overridden By Rate Limit)' print('[+] Delay Between Requests: {} \n'.format(request_delay)) def run(self): last_run = round(time.time()) while True: os.system('cls') self.print_current_stats() self.print_current_settings() self.print_api_stats() if round(time.time()) - last_run > self.delay_between_requests: self.insert_latest_images() self.flush_failed_votes_and_comments() last_run = round(time.time()) self._check_thread_status() time.sleep(2)
from imgurpython import ImgurClient client_id = '51a86b471a85443' client_secret = '442df4f265e082d2c168ad2ea7da3607b12b37e8' target_file = str(sys.argv[1]) client = ImgurClient(client_id, client_secret) target = open(target_file, 'w') # Example request for i in range(10): items = client.gallery(section='hot', sort='viral', page=i, window='day', show_viral=True) for item in items: if ".jpg" in item.link: target.write(str(item.link)) target.write("\n") print item.link target.close()
from imgurpython import ImgurClient PATH = "imgur1.txt" client_id = 'b68fbc6850008f8' client_secret = 'b83cc295315d15e167a61d3a8a572ee5bf42d731' imgur_client = ImgurClient(client_id, client_secret) imgur_gallery = imgur_client.gallery() # CMU 15-112 def writeFile(filename, contents, mode="wt"): with open(filename, mode) as fout: fout.write(contents) def main(): result = "" items = imgur_gallery extensionIndex = -4 # get random images for item in items: # image link if item.link[extensionIndex] == ".": result += item.link + "\n" writeFile(PATH, result) if __name__ == "__main__": main()
from imgurpython import ImgurClient import json client_id = '##############' client_secret = '#######################' client = ImgurClient(client_id, client_secret) def convert_list_to_string(mlist): converted_list = str(mlist).strip('[]') converted_list = converted_list.replace('\'', '') return converted_list image_list = [] # Example request for i in range(0, 50): print("PAGE: " + str(i)) items = client.gallery(page=0) for item in items: image_list.append(item.link) json_out = {"Images": convert_list_to_string(image_list)} with open('images.json', 'w') as outfile: json.dump(json_out, outfile)