def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1):
if 'tumblr' in oldpath:
# Can't properly handle tumblr links
self.debug('cannot properly handle tumblr links; trying anyway')
#return
if subdir == '' and album_id == -1:
self.debug('adding image: %s' % oldpath)
# Ensure image is an actual image
try:
dims = ImageUtils.get_dimensions(oldpath)
except:
self.debug('failed to load image: %s, skipping' % oldpath)
return
newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage)
newimage = newimage.replace('.jpeg.jpg', '.jpg')
thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage)
thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg')
if path.exists(newimage):
self.debug('new image already exists: %s' % newimage)
return
ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs'))
copy2(oldpath, newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception, e:
self.debug('failed to create thumbnail: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
def start(self):
'''
Overriding SiteBase's start() method for unique ripping logic
'''
# We need a lot of libraries
from ImageUtils import ImageUtils
from calendar import timegm
from shutil import copy2, rmtree
from time import gmtime
from os import path, walk, environ, getcwd
from json import loads
savedir = path.join('rips', self.path)
if getcwd().endswith('py'):
savedir = path.join('..', savedir)
if self.album_exists:
# Don't re-rip an album. Return info about existing album.
return {
'warning' : 'album already exists',
'album_id' : self.album_id,
'album' : self.album_name,
'url' : self.url,
'host' : self.get_host(),
'path' : self.path,
'count' : self.db.count('medias', 'album_id = ?', [self.album_id]),
'pending' : self.db.count('urls', 'album_id = ?', [self.album_id])
}
user = self.url.split(':')[-1]
# Search for username (with proper case) on site
gwapi = self.db.get_config('gw_api')
if gwapi == None:
raise Exception('unable to rip gonewild albums: gw_api is null')
r = self.httpy.get('%s?method=search_user&user=%s' % (gwapi, user))
json = loads(r)
found = False
for jsonuser in json['users']:
if jsonuser.lower() == user.lower():
found = True
user = jsonuser
break
gwroot = self.db.get_config('gw_root')
if gwroot == None:
raise Exception('unable to rip gonewild albums: gw_root is null')
userroot = path.join(gwroot, user)
# Check if we can actually rip this user
if not found or not path.exists(userroot):
return {
'error' : 'unable to rip user (not archived)'
}
# Create subdirs
ImageUtils.create_subdirectories(path.join(savedir, 'thumbs'))
# Copy images to /rips/, get values that need to be inserted into db (insertmany)
insertmany = []
already_got = []
filesize = 0
for root, subdirs, files in walk(userroot):
if root.endswith('thumbs'): continue
for filename in sorted(files):
f = path.join(root, filename)
n = filename
if not root.endswith(userroot):
# It's a subidr, save the file accordingly
n = '%s_%s' % (root[root.rfind('/')+1:], filename)
# Avoid duplicates
no_post = n[n.rfind('_')+1:]
if no_post in already_got: continue
already_got.append(no_post)
n = '%03d_%s' % (len(insertmany) + 1, n)
saveas = path.join(savedir, n)
# Copy & get size
try:
copy2(f, saveas)
(width, height) = ImageUtils.get_dimensions(saveas)
except Exception, e:
# image can't be parsed, probably corrupt. move on.
continue
# Create thumbnail
tsaveas = path.join(savedir, 'thumbs', n)
try:
(tsaveas, twidth, theight) = ImageUtils.create_thumbnail(saveas, tsaveas)
except Exception, e:
# Failed to create thumb
tsaveas = '/'.join(['ui', 'images', 'nothumb.png'])
twidth = theight = 160
filesize += path.getsize(saveas)
# Add to list of values to insert into DB
insertmany.append( [
self.album_id, # album_id, currently None
len(insertmany) + 1, # i_index
'', # url TODO
1, # valid
None, # error
SiteBase.get_type(saveas), # type
n, # image_name
width, # img width
height, # img height
path.getsize(saveas), # filesize
path.basename(tsaveas), # thumb_name
twidth, # thumb width
theight, # thumb height
None # metadata
] )
if path.exists(newimage):
self.debug('new image already exists: %s' % newimage)
return
ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs'))
copy2(oldpath, newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception, e:
self.debug('failed to create thumbnail: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
(post, comment, imgid) = self.get_post_comment_id(oldimage)
url = 'http://i.imgur.com/%s' % imgid
dims = ImageUtils.get_dimensions(newimage)
size = path.getsize(newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception, e:
self.debug('add_existing_image: create_thumbnail failed: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
try:
self.add_image(newimage, user, url,
dims[0], dims[1], size, thumbnail, 'image',
album_id, post, comment)
except Exception, e:
self.debug('add_existing_image: failed: %s' % str(e))
return
if subdir == '' and album_id == -1: # Not an album
# Download URL
try:
self.debug('%s: process_url: downloading #%d %s' %
(child.author, media_index + 1, media))
ImageUtils.httpy.download(media, saveas)
if path.getsize(saveas) == 503:
raise Exception('503b = removed')
except Exception, e:
self.debug(
'%s: process_url: failed to download #%d: %s, moving on' %
(child.author, media_index + 1, str(e)))
continue
# Get media information (width, height, size)
try:
(width, height) = ImageUtils.get_dimensions(saveas)
except Exception, e:
# If we cannot process the media file, skip it!
self.debug('%s: process_url: #%d %s' %
(child.author, media_index + 1, str(e)))
continue
size = path.getsize(saveas)
# Create thumbnail
savethumbas = path.join(working_dir, 'thumbs', fname)
try:
savethumbas = ImageUtils.create_thumbnail(saveas, savethumbas)
except Exception, e:
savethumbas = path.join(ImageUtils.get_root(), 'images',
'nothumb.png')
self.debug(
print "THREAD: %s: failed to download %s to %s: %s\n%s" % (
url["path"],
url["url"],
saveas,
str(e),
str(format_exc()),
)
result["error"] = "failed to download %s to %s: %s\n%s" % (url["url"], saveas, str(e), str(format_exc()))
self.results.append(result)
self.current_threads.pop()
return
# Save image info
result["filesize"] = path.getsize(saveas)
try:
(result["width"], result["height"]) = ImageUtils.get_dimensions(saveas)
except Exception, e:
# This fails if we can't identify the image file. Consider it errored
print "THREAD: %s: failed to identify image file %s from %s: %s\n%s" % (
url["path"],
saveas,
url["url"],
str(e),
format_exc(),
)
result["error"] = "failed to identify image file %s from %s: %s\n%s" % (
saveas,
url["url"],
str(e),
format_exc(),
)
}
ImageUtils.httpy.download(media, saveas, headers=headers)
if path.getsize(saveas) == 503:
raise Exception('503b = removed')
except Exception, e:
self.debug('%s: process_url: failed to download #%d: %s, moving on' % (child.author, media_index + 1, str(e)))
continue
# Get media information (width, height, thumbsaveas)
if media_type == 'audio':
# Audio files don't have width/height/thumbnail
width = height = 0
savethumbas = path.join(ImageUtils.get_root(), 'images', 'audio.png')
else:
try:
(width, height) = ImageUtils.get_dimensions(saveas)
except Exception, e:
# If we cannot process the media file, skip it!
self.debug('%s: process_url: #%d %s' % (child.author, media_index + 1, str(e)))
continue
# Create thumbnail if needed
if self.db.get_config('save_thumbnails', 'true') == 'false':
savethumbas = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
else:
savethumbas = path.join(working_dir, 'thumbs', fname)
try:
savethumbas = ImageUtils.create_thumbnail(saveas, savethumbas)
except Exception, e:
savethumbas = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
self.debug('%s: process_url: failed to create thumb #%d: %s, using default' % (child.author, media_index + 1, str(e)))
def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1):
if 'tumblr' in oldpath:
# Can't properly handle tumblr links
self.debug('cannot properly handle tumblr links; trying anyway')
#return
if subdir == '' and album_id == -1:
self.debug('adding image: %s' % oldpath)
# Ensure image is an actual image
try:
dims = ImageUtils.get_dimensions(oldpath)
except:
self.debug('failed to load image: %s, skipping' % oldpath)
return
newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage)
newimage = newimage.replace('.jpeg.jpg', '.jpg')
thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage)
thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg')
if path.exists(newimage):
self.debug('new image already exists: %s' % newimage)
return
ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs'))
copy2(oldpath, newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception as e:
self.debug('failed to create thumbnail: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
(post, comment, imgid) = self.get_post_comment_id(oldimage)
url = 'http://i.imgur.com/%s' % imgid
dims = ImageUtils.get_dimensions(newimage)
size = path.getsize(newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception as e:
self.debug('add_existing_image: create_thumbnail failed: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
try:
self.add_image(newimage, user, url,
dims[0], dims[1], size, thumbnail, 'image',
album_id, post, comment)
except Exception as e:
self.debug('add_existing_image: failed: %s' % str(e))
return
if subdir == '' and album_id == -1: # Not an album
# Add post
p = Post()
p.id = post
p.author = user
if comment == None: p.url = url
p.created = path.getctime(oldpath)
p.subreddit = ''
p.title = ''
try:
self.add_post(p, legacy=1)
except Exception as e:
self.debug('add_existing_image: create post failed: %s' % str(e))
# Add comment
if comment != None:
c = Comment()
c.id = comment
c.post_id = post
c.author = user
if comment != None: c.body = url
p.created = path.getctime(oldpath)
try:
self.add_comment(c, legacy=1)
except Exception as e:
self.debug('add_existing_image: create comment failed: %s' % str(e))
result['image_name'] = path.basename(saveas) # Attempt to dowload image at URL try: self.httpy.download(url['url'], saveas) except Exception, e: print 'THREAD: %s: failed to download %s to %s: %s\n%s' % (url['path'], url['url'], saveas, str(e), str(format_exc())) result['error'] = 'failed to download %s to %s: %s\n%s' % (url['url'], saveas, str(e), str(format_exc())) self.to_remove.append( (url['album_id'], url['i_index'] ) ) self.results.append(result) self.current_threads.pop() return # Save image info result['filesize'] = path.getsize(saveas) try: (result['width'], result['height']) = ImageUtils.get_dimensions(saveas) except Exception, e: # This fails if we can't identify the image file. Consider it errored print 'THREAD: %s: failed to identify image file %s from %s: %s\n%s' % (url['path'], saveas, url['url'], str(e), format_exc()) result['error'] = 'failed to identify image file %s from %s: %s\n%s' % (saveas, url['url'], str(e), format_exc()) self.to_remove.append( (url['album_id'], url['i_index'] ) ) self.results.append(result) self.current_threads.pop() return # Get thumbnail tsaveas = path.join(dirname, 'thumbs', url['saveas']) try: (tsaveas, result['t_width'], result['t_height']) = ImageUtils.create_thumbnail(saveas, tsaveas) except Exception, e: # Failed to create thumbnail, use default
