def test_gifv_gif_direct_link(): # since the extension is grabbed directly from the url, .gifv is the initial extension imgur = ImgurDownloader('http://i.imgur.com/MOvVbhc.gifv') # since the media is natively a video (.mp4) it is saved as such # NOTE: media downloaded is 1.4 MB imgur.save_images() file = os.path.join(os.getcwd(), 'MOvVbhc.mp4') assert(os.path.isfile(file)) os.remove(file)
async def download_album_for_guild(guild_id, url): for file in os.listdir(os.path.join(os.path.curdir, IMGUR_DOWNLOAD_PATH, str(guild_id))): file_path = os.path.join(os.path.curdir, IMGUR_DOWNLOAD_PATH, str(guild_id), file) try: if os.path.isfile(file_path): os.unlink(file_path) except Exception as e: print(e) return ImgurDownloader(url, os.path.join(os.path.curdir, IMGUR_DOWNLOAD_PATH), file_name=str(guild_id), delete_dne=True).save_images()
def process_candidate(current_candidate, send_to_email=True): """ Main function that generates the sign, video, and speech using the helper functions, then sends the campaign in a zip over email set send_to_email to false to disable emailing functionality""" # Create the campaign sign # Get the candidate's image from Imgur ImgurDownloader(current_candidate["image"]).save_images(CWD("candidate_images")) # Download Imgur image/album candidate_image = CWD(f"candidate_images/{os.listdir('candidate_images')[0]}") # Find the first image in the album # Generate the sign with the candidates data print(f"Generating campaign sign for {current_candidate['name']}") sign_output = CWD(f"output/{current_candidate['name']} sign.png") generate_sign(current_candidate["name"], current_candidate["position"], current_candidate["district"], current_candidate["party"], candidate_image, sign_output) rmtree(CWD("candidate_images")) # Delete the images folder # Create the campaign speech speech = f"""My Fellow Americans, I stand before you today not to spew fancy political rhetoric or to talk about what could be better. I’m here today because you all are here. And you’re clearly fed up with politics. I’m here because our democracy isn’t working for all of us. You all know that this is the most important election of our lifetime. So I’m going to do something. This is why I, {current_candidate['name']}, am running to be the {current_candidate['position']} of {current_candidate['district']}. You may think more divides us than unites us in this new era of politics. But I know we can all unite behind {current_candidate['issues'][0]}. Not only do our lives depend on making progress on {current_candidate['issues'][0]}, but so do the lives of our children. However, putting our all towards {current_candidate['issues'][0]} won’t be enough to save our democracy. I know each and every single one you is struggling. That’s why I am also going to also focus on {current_candidate['issues'][1]} and {current_candidate['issues'][2]}. The elites in power feel too comfortable, so we need to shake things up. I am here today to listen to you, the American people, because I know you have been silenced for far too long. Vote {current_candidate['name']} for {current_candidate['position']} of {current_candidate['district']}! """ # Write the speech to a file with open(CWD(f"output/{current_candidate['name']} speech.txt"), "w") as file: file.write(speech) generate_video(sign_output, current_candidate["issues"][0], CWD(f"output/{current_candidate['name']} video.mp4")) # Zip all files in output folder file_paths = os.listdir(CWD("output")) with ZipFile(os.path.join(CWD('output'), 'Generated Campaign.zip'), 'w') as zipped: # writing each file one by one os.chdir(CWD("output")) for file in file_paths: if "zip" not in file and current_candidate["name"] in file: zipped.write(file, arcname=file) os.remove(file) zipped.close() if send_to_email: send_email(current_candidate["email"], "Generated Campaign.zip") if os.path.exists(f"{current_candidate['name']}.zip"): os.remove(f"{current_candidate['name']}.zip") os.rename("Generated Campaign.zip", f"{current_candidate['name']}.zip") os.chdir("..")
def test_image_ids_and_extensions(url, exp_id): imgur = ImgurDownloader(url) # assert(exp_id == imgur.json_imageIDs) assert(exp_id == imgur.imageIDs)
def download_images(image_links: List[str]): os.makedirs(os.path.join(this_dir, "wallpapers"), exist_ok=True) couldnt_download = open(os.path.join(this_dir, "failed.txt"), "a") ydl = YoutubeDL({"outtmpl": "./wallpapers/%(title)s.%(ext)s"}) def_headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' } for i, url in enumerate(image_links, 1): click.secho(f"({i}/{len(image_links)}) Trying to download: '{url}'...", fg="blue") if "old.reddit.com" in url: try: # youtube-dl can download reddit videos # it will fallback to parsing the page/downloading the content anyways # so this works for any images which are not direct links to the image ydl.download([url]) click.secho("Download succeeded!", fg="green") random_wait() except: click.secho(f"Couldn't download '{url}'.", fg="red", err=True) couldnt_download.write(f"{url}\n") couldnt_download.flush() wait_for_internet() elif "imgur" in url: try: # this may fail if a URL not on imgur has imgur in the url, but I'm fine ignoring that # case and going through the URL manually after its written to failed.txt ImgurDownloader( url, dir_download=os.path.join(this_dir, "wallpapers"), debug=True, delete_dne=False, ).save_images() click.secho("Download succeeded!", fg="green") random_wait() except Exception as e: print(str(e)) click.secho(f"Couldn't download '{url}'.", fg="red", err=True) couldnt_download.write(f"{url}\n") couldnt_download.flush() wait_for_internet() else: try: r = requests.get(url, stream=True, headers=def_headers) try: Image.open(BytesIO(r.content)).save( "./wallpapers/{}.png".format(datetime.now())) click.secho("Download succeeded!", fg="green") except OSError as oe: # image failed to be parsed click.echo(str(oe)) raise oe # re-raise so that the failed image decode gets written to failed.txt random_wait() except: click.secho(f"Couldn't download '{url}'.", fg="red", err=True) couldnt_download.write(f"{url}\n") couldnt_download.flush() wait_for_internet() couldnt_download.close()
print(f'Logged in as {reddit_api.GetUsername()}') q = queue.Queue() extractor = URLExtract() table = {} while(True): for submission in reddit_api.GetNewSubmissions(): if(submission.id not in table): print(f'{submission.title} by {submission.author}') table[submission.id] = True q.put(submission.id) if(q.qsize() > 100): item_to_delete = q.get() table.pop(item_to_delete) urls = extractor.find_urls(submission.selftext) print(urls) for match in urls: if('imgur' in match): print(match) try: ImgurDownloader(match, dir_download='F:/Timestamps', delete_dne=True).save_images() except: print("failed dl") sleep(60)
def download(self): """Download media from submissions""" continue_downloading = True # var limit is constant, self.limit is not constant limit = self.limit # counters to keep track of how many submissions downloaded & more download_count, error_count, skip_count = 0, 0, 0 # load last-id of submission downloaded from or create new file for id log_filename = '._history.txt' log_data, prev_id = process_subreddit_last_id(subreddit=self.subreddit, sort_type=self.sort_type, dir=self.path, log_file=log_filename, verbose=True) if not self.previous_id: self.set_previous_id(prev_id) # ensures the amount of submissions downloaded from is equal to limit while continue_downloading: errors, skips = 0, 0 # get submissions (dict containing info) & use data to download submissions = self.get_submissions_info() for submission in submissions: url = submission['url'] title = submission['title'] # makes an assumption that len(file_extension) <= 5 _, filename = shorten_file_path_if_needed( slugify(title), max_length=self.OS_MAX_PATH_LENGTH - len(self.path) - 5) dl_directory = submission['dl_directory'] submission_id = submission['id'] # filename is '' or None, filename = datetime.now() if not filename: _, filename = shorten_file_path_if_needed( get_datetime_now(), max_length=self.OS_MAX_PATH_LENGTH - len(self.path) - 5) # if an entire imgur album was downloaded, # filenames will be stored here final_filenames = [] self.log.info('Attempting to save {} as {}'.format( url, dl_directory)) # check domain and call corresponding downloader # download functions or methods try: if 'imgur.com' in url: imgur = ImgurDownloader(imgur_url=url, dir_download=self.path, file_name=filename, delete_dne=True, debug=False) final_filenames, skipped = imgur.save_images() if len(final_filenames) == 1: filename = final_filenames[0] dl_directory = os.path.join( os.path.dirname(dl_directory), filename) elif 'deviantart.com' in url: download_deviantart_url(url, dl_directory) else: job = DownloadJob(url) job.run() # text submission on a subreddit if job.pathfmt is None: raise TurboPalmTreeException( 'No path for gallery-dl DownloadJob\n' '\turl = {}'.format(url)) dl_directory = os.path.abspath(job.pathfmt.path) dl_directory = move_file( dl_directory, join(self.path, filename + get_file_extension(dl_directory))) print('downloaded: {title}; {url}'.format(title=filename, url=url)) # get time if file is created, else just use the time now if dl_directory and os.path.exists(dl_directory): creation_time = os.path.getctime(dl_directory) else: creation_time = time.time() if not self.disable_im: metadata = { 'source_url': url, 'creation_time': creation_time } # add img, locate & delete older duplicates self.im.delete_duplicates(dl_directory, metadata=metadata) if not self.disable_db: # add some data to dict insert data into database submission['download_date'] = convert_to_readable_time( creation_time) self.db.insert(submission) except self.Exceptions as e: msg = '{}: {}'.format(type(e).__name__, e.args) self.log.warning(msg) print(Fore.RED + msg + Style.RESET_ALL) errors += 1 except KeyboardInterrupt: msg = 'KeyboardInterrupt caught, exiting program' self.log.info(msg) print(msg) continue_downloading = False break # update previous id downloaded if 'submission_id' in locals().keys(): self.set_previous_id(submission_id) # update count of media successfully downloaded download_count += self.limit - errors - skips error_count += errors skip_count += skips # update attribute limit which is used when getting submissions if download_count < limit: self.set_limit(limit - download_count) elif download_count >= limit or not continue_downloading: if 'submission_id' in locals().keys(): log_data[self.subreddit][self.sort_type]['last-id'] = \ submission_id history_log(self.path, log_filename, 'write', log_data) continue_downloading = False # continue_downloading is false if not self.disable_db: self.db.close() self._cleanup_files() print("{}{} errors occured".format(Fore.YELLOW, error_count)) print("{}Downloaded from {} submissions from {}/{}{reset}".format( Fore.GREEN, download_count, self.subreddit, self.sort_type, reset=Style.RESET_ALL))
def doDownload(): global downloadQueue global currentDownloadUrl global loopBreaker global terminateFlag global imgurAlbumSize print(downloadFormatString) ydl_opts = { "logger": MyLogger(), "progress_hooks": [my_hook], "prefer_ffmpeg": True, "restrictfilenames": True, "format": downloadFormatString, } nextUrl = getNextQueuedItem() if nextUrl != "NONE": currentDownloadUrl = nextUrl["url"] print("proceeding to " + currentDownloadUrl) try: # there's a bug where this will error if your download folder is inside your application folder os.chdir(youtubelocation) match = re.match( "(https?)://(www\.)?(i\.|m\.)?imgur\.com/(a/|gallery/|r/)?/?(\w*)/?(\w*)(#[0-9]+)?(.\w*)?", # NOQA currentDownloadUrl, ) if match: downloadQueue[currentDownloadUrl]["status"] = "downloading" downloadQueue[currentDownloadUrl]["mode"] = "imgur" print("Matched Regex") downloader = ImgurDownloader(currentDownloadUrl, youtubelocation) print("Downloader created...") print("This albums has {} images".format( downloader.num_images())) imgurAlbumSize = downloader.num_images() downloader.on_image_download(imgurOnDownloadHook) resultsTuple = downloader.save_images() print("Saved!") print(resultsTuple) downloadQueue[currentDownloadUrl]["status"] = "completed" if not match: nextUrl["mode"] = "youtube" with youtube_dl.YoutubeDL(ydl_opts) as ydl: ydl.download([nextUrl["url"]]) downloadQueue[nextUrl["url"]]["status"] = "completed" downloadQueue[nextUrl["url"]]["playable"] = queryVideo( downloadQueue[nextUrl["url"]]["filename"]) os.chdir(os.path.dirname(os.path.realpath(__file__))) loopBreaker = 10 except Exception as e: nextUrl["status"] = "error" nextUrl["error"] = e os.chdir(os.path.dirname(os.path.realpath(__file__))) nextUrl = getNextQueuedItem() if nextUrl != "NONE" and loopBreaker > 0: loopBreaker = loopBreaker - 1 print("loopBreaker:" + str(loopBreaker)) if terminateFlag == 0: doDownload() else: print("Nothing to do - Finishing Process")