def test_gifv_gif_direct_link():
# since the extension is grabbed directly from the url, .gifv is the initial extension
imgur = ImgurDownloader('http://i.imgur.com/MOvVbhc.gifv')
# since the media is natively a video (.mp4) it is saved as such
# NOTE: media downloaded is 1.4 MB
imgur.save_images()
file = os.path.join(os.getcwd(), 'MOvVbhc.mp4')
assert(os.path.isfile(file))
os.remove(file)
async def download_album_for_guild(guild_id, url):
for file in os.listdir(os.path.join(os.path.curdir, IMGUR_DOWNLOAD_PATH, str(guild_id))):
file_path = os.path.join(os.path.curdir, IMGUR_DOWNLOAD_PATH, str(guild_id), file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as e:
print(e)
return ImgurDownloader(url, os.path.join(os.path.curdir, IMGUR_DOWNLOAD_PATH), file_name=str(guild_id), delete_dne=True).save_images()
def process_candidate(current_candidate, send_to_email=True):
""" Main function that generates the sign, video, and speech using the helper functions,
then sends the campaign in a zip over email
set send_to_email to false to disable emailing functionality"""
# Create the campaign sign
# Get the candidate's image from Imgur
ImgurDownloader(current_candidate["image"]).save_images(CWD("candidate_images")) # Download Imgur image/album
candidate_image = CWD(f"candidate_images/{os.listdir('candidate_images')[0]}") # Find the first image in the album
# Generate the sign with the candidates data
print(f"Generating campaign sign for {current_candidate['name']}")
sign_output = CWD(f"output/{current_candidate['name']} sign.png")
generate_sign(current_candidate["name"], current_candidate["position"],
current_candidate["district"], current_candidate["party"],
candidate_image, sign_output)
rmtree(CWD("candidate_images")) # Delete the images folder
# Create the campaign speech
speech = f"""My Fellow Americans,
I stand before you today not to spew fancy political rhetoric or to talk about what could be better. I’m here today because you all are here. And you’re clearly fed up with politics. I’m here because our democracy isn’t working for all of us. You all know that this is the most important election of our lifetime. So I’m going to do something. This is why I, {current_candidate['name']}, am running to be the {current_candidate['position']} of {current_candidate['district']}.
You may think more divides us than unites us in this new era of politics. But I know we can all unite behind {current_candidate['issues'][0]}. Not only do our lives depend on making progress on {current_candidate['issues'][0]}, but so do the lives of our children.
However, putting our all towards {current_candidate['issues'][0]} won’t be enough to save our democracy. I know each and every single one you is struggling. That’s why I am also going to also focus on {current_candidate['issues'][1]} and {current_candidate['issues'][2]}. The elites in power feel too comfortable, so we need to shake things up. I am here today to listen to you, the American people, because I know you have been silenced for far too long.
Vote {current_candidate['name']} for {current_candidate['position']} of {current_candidate['district']}!
"""
# Write the speech to a file
with open(CWD(f"output/{current_candidate['name']} speech.txt"), "w") as file:
file.write(speech)
generate_video(sign_output, current_candidate["issues"][0],
CWD(f"output/{current_candidate['name']} video.mp4"))
# Zip all files in output folder
file_paths = os.listdir(CWD("output"))
with ZipFile(os.path.join(CWD('output'), 'Generated Campaign.zip'), 'w') as zipped:
# writing each file one by one
os.chdir(CWD("output"))
for file in file_paths:
if "zip" not in file and current_candidate["name"] in file:
zipped.write(file, arcname=file)
os.remove(file)
zipped.close()
if send_to_email:
send_email(current_candidate["email"], "Generated Campaign.zip")
if os.path.exists(f"{current_candidate['name']}.zip"):
os.remove(f"{current_candidate['name']}.zip")
os.rename("Generated Campaign.zip", f"{current_candidate['name']}.zip")
os.chdir("..")
def test_image_ids_and_extensions(url, exp_id):
imgur = ImgurDownloader(url)
# assert(exp_id == imgur.json_imageIDs)
assert(exp_id == imgur.imageIDs)
def download_images(image_links: List[str]):
os.makedirs(os.path.join(this_dir, "wallpapers"), exist_ok=True)
couldnt_download = open(os.path.join(this_dir, "failed.txt"), "a")
ydl = YoutubeDL({"outtmpl": "./wallpapers/%(title)s.%(ext)s"})
def_headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
}
for i, url in enumerate(image_links, 1):
click.secho(f"({i}/{len(image_links)}) Trying to download: '{url}'...",
fg="blue")
if "old.reddit.com" in url:
try:
# youtube-dl can download reddit videos
# it will fallback to parsing the page/downloading the content anyways
# so this works for any images which are not direct links to the image
ydl.download([url])
click.secho("Download succeeded!", fg="green")
random_wait()
except:
click.secho(f"Couldn't download '{url}'.", fg="red", err=True)
couldnt_download.write(f"{url}\n")
couldnt_download.flush()
wait_for_internet()
elif "imgur" in url:
try:
# this may fail if a URL not on imgur has imgur in the url, but I'm fine ignoring that
# case and going through the URL manually after its written to failed.txt
ImgurDownloader(
url,
dir_download=os.path.join(this_dir, "wallpapers"),
debug=True,
delete_dne=False,
).save_images()
click.secho("Download succeeded!", fg="green")
random_wait()
except Exception as e:
print(str(e))
click.secho(f"Couldn't download '{url}'.", fg="red", err=True)
couldnt_download.write(f"{url}\n")
couldnt_download.flush()
wait_for_internet()
else:
try:
r = requests.get(url, stream=True, headers=def_headers)
try:
Image.open(BytesIO(r.content)).save(
"./wallpapers/{}.png".format(datetime.now()))
click.secho("Download succeeded!", fg="green")
except OSError as oe: # image failed to be parsed
click.echo(str(oe))
raise oe # re-raise so that the failed image decode gets written to failed.txt
random_wait()
except:
click.secho(f"Couldn't download '{url}'.", fg="red", err=True)
couldnt_download.write(f"{url}\n")
couldnt_download.flush()
wait_for_internet()
couldnt_download.close()
print(f'Logged in as {reddit_api.GetUsername()}')
q = queue.Queue()
extractor = URLExtract()
table = {}
while(True):
for submission in reddit_api.GetNewSubmissions():
if(submission.id not in table):
print(f'{submission.title} by {submission.author}')
table[submission.id] = True
q.put(submission.id)
if(q.qsize() > 100):
item_to_delete = q.get()
table.pop(item_to_delete)
urls = extractor.find_urls(submission.selftext)
print(urls)
for match in urls:
if('imgur' in match):
print(match)
try:
ImgurDownloader(match, dir_download='F:/Timestamps', delete_dne=True).save_images()
except:
print("failed dl")
sleep(60)
def download(self):
"""Download media from submissions"""
continue_downloading = True
# var limit is constant, self.limit is not constant
limit = self.limit
# counters to keep track of how many submissions downloaded & more
download_count, error_count, skip_count = 0, 0, 0
# load last-id of submission downloaded from or create new file for id
log_filename = '._history.txt'
log_data, prev_id = process_subreddit_last_id(subreddit=self.subreddit,
sort_type=self.sort_type,
dir=self.path,
log_file=log_filename,
verbose=True)
if not self.previous_id:
self.set_previous_id(prev_id)
# ensures the amount of submissions downloaded from is equal to limit
while continue_downloading:
errors, skips = 0, 0
# get submissions (dict containing info) & use data to download
submissions = self.get_submissions_info()
for submission in submissions:
url = submission['url']
title = submission['title']
# makes an assumption that len(file_extension) <= 5
_, filename = shorten_file_path_if_needed(
slugify(title),
max_length=self.OS_MAX_PATH_LENGTH - len(self.path) - 5)
dl_directory = submission['dl_directory']
submission_id = submission['id']
# filename is '' or None, filename = datetime.now()
if not filename:
_, filename = shorten_file_path_if_needed(
get_datetime_now(),
max_length=self.OS_MAX_PATH_LENGTH - len(self.path) -
5)
# if an entire imgur album was downloaded,
# filenames will be stored here
final_filenames = []
self.log.info('Attempting to save {} as {}'.format(
url, dl_directory))
# check domain and call corresponding downloader
# download functions or methods
try:
if 'imgur.com' in url:
imgur = ImgurDownloader(imgur_url=url,
dir_download=self.path,
file_name=filename,
delete_dne=True,
debug=False)
final_filenames, skipped = imgur.save_images()
if len(final_filenames) == 1:
filename = final_filenames[0]
dl_directory = os.path.join(
os.path.dirname(dl_directory), filename)
elif 'deviantart.com' in url:
download_deviantart_url(url, dl_directory)
else:
job = DownloadJob(url)
job.run()
# text submission on a subreddit
if job.pathfmt is None:
raise TurboPalmTreeException(
'No path for gallery-dl DownloadJob\n'
'\turl = {}'.format(url))
dl_directory = os.path.abspath(job.pathfmt.path)
dl_directory = move_file(
dl_directory,
join(self.path,
filename + get_file_extension(dl_directory)))
print('downloaded: {title}; {url}'.format(title=filename,
url=url))
# get time if file is created, else just use the time now
if dl_directory and os.path.exists(dl_directory):
creation_time = os.path.getctime(dl_directory)
else:
creation_time = time.time()
if not self.disable_im:
metadata = {
'source_url': url,
'creation_time': creation_time
}
# add img, locate & delete older duplicates
self.im.delete_duplicates(dl_directory,
metadata=metadata)
if not self.disable_db:
# add some data to dict insert data into database
submission['download_date'] = convert_to_readable_time(
creation_time)
self.db.insert(submission)
except self.Exceptions as e:
msg = '{}: {}'.format(type(e).__name__, e.args)
self.log.warning(msg)
print(Fore.RED + msg + Style.RESET_ALL)
errors += 1
except KeyboardInterrupt:
msg = 'KeyboardInterrupt caught, exiting program'
self.log.info(msg)
print(msg)
continue_downloading = False
break
# update previous id downloaded
if 'submission_id' in locals().keys():
self.set_previous_id(submission_id)
# update count of media successfully downloaded
download_count += self.limit - errors - skips
error_count += errors
skip_count += skips
# update attribute limit which is used when getting submissions
if download_count < limit:
self.set_limit(limit - download_count)
elif download_count >= limit or not continue_downloading:
if 'submission_id' in locals().keys():
log_data[self.subreddit][self.sort_type]['last-id'] = \
submission_id
history_log(self.path, log_filename, 'write', log_data)
continue_downloading = False
# continue_downloading is false
if not self.disable_db:
self.db.close()
self._cleanup_files()
print("{}{} errors occured".format(Fore.YELLOW, error_count))
print("{}Downloaded from {} submissions from {}/{}{reset}".format(
Fore.GREEN,
download_count,
self.subreddit,
self.sort_type,
reset=Style.RESET_ALL))
def doDownload():
global downloadQueue
global currentDownloadUrl
global loopBreaker
global terminateFlag
global imgurAlbumSize
print(downloadFormatString)
ydl_opts = {
"logger": MyLogger(),
"progress_hooks": [my_hook],
"prefer_ffmpeg": True,
"restrictfilenames": True,
"format": downloadFormatString,
}
nextUrl = getNextQueuedItem()
if nextUrl != "NONE":
currentDownloadUrl = nextUrl["url"]
print("proceeding to " + currentDownloadUrl)
try:
# there's a bug where this will error if your download folder is inside your application folder
os.chdir(youtubelocation)
match = re.match(
"(https?)://(www\.)?(i\.|m\.)?imgur\.com/(a/|gallery/|r/)?/?(\w*)/?(\w*)(#[0-9]+)?(.\w*)?", # NOQA
currentDownloadUrl,
)
if match:
downloadQueue[currentDownloadUrl]["status"] = "downloading"
downloadQueue[currentDownloadUrl]["mode"] = "imgur"
print("Matched Regex")
downloader = ImgurDownloader(currentDownloadUrl,
youtubelocation)
print("Downloader created...")
print("This albums has {} images".format(
downloader.num_images()))
imgurAlbumSize = downloader.num_images()
downloader.on_image_download(imgurOnDownloadHook)
resultsTuple = downloader.save_images()
print("Saved!")
print(resultsTuple)
downloadQueue[currentDownloadUrl]["status"] = "completed"
if not match:
nextUrl["mode"] = "youtube"
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([nextUrl["url"]])
downloadQueue[nextUrl["url"]]["status"] = "completed"
downloadQueue[nextUrl["url"]]["playable"] = queryVideo(
downloadQueue[nextUrl["url"]]["filename"])
os.chdir(os.path.dirname(os.path.realpath(__file__)))
loopBreaker = 10
except Exception as e:
nextUrl["status"] = "error"
nextUrl["error"] = e
os.chdir(os.path.dirname(os.path.realpath(__file__)))
nextUrl = getNextQueuedItem()
if nextUrl != "NONE" and loopBreaker > 0:
loopBreaker = loopBreaker - 1
print("loopBreaker:" + str(loopBreaker))
if terminateFlag == 0:
doDownload()
else:
print("Nothing to do - Finishing Process")