def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1): if 'tumblr' in oldpath: # Can't properly handle tumblr links self.debug('cannot properly handle tumblr links; trying anyway') #return if subdir == '' and album_id == -1: self.debug('adding image: %s' % oldpath) # Ensure image is an actual image try: dims = ImageUtils.get_dimensions(oldpath) except: self.debug('failed to load image: %s, skipping' % oldpath) return newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage) newimage = newimage.replace('.jpeg.jpg', '.jpg') thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage) thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg') if path.exists(newimage): self.debug('new image already exists: %s' % newimage) return ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs')) copy2(oldpath, newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception, e: self.debug('failed to create thumbnail: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
def start(self): ''' Overriding SiteBase's start() method for unique ripping logic ''' # We need a lot of libraries from ImageUtils import ImageUtils from calendar import timegm from shutil import copy2, rmtree from time import gmtime from os import path, walk, environ, getcwd from json import loads savedir = path.join('rips', self.path) if getcwd().endswith('py'): savedir = path.join('..', savedir) if self.album_exists: # Don't re-rip an album. Return info about existing album. return { 'warning' : 'album already exists', 'album_id' : self.album_id, 'album' : self.album_name, 'url' : self.url, 'host' : self.get_host(), 'path' : self.path, 'count' : self.db.count('medias', 'album_id = ?', [self.album_id]), 'pending' : self.db.count('urls', 'album_id = ?', [self.album_id]) } user = self.url.split(':')[-1] # Search for username (with proper case) on site gwapi = self.db.get_config('gw_api') if gwapi == None: raise Exception('unable to rip gonewild albums: gw_api is null') r = self.httpy.get('%s?method=search_user&user=%s' % (gwapi, user)) json = loads(r) found = False for jsonuser in json['users']: if jsonuser.lower() == user.lower(): found = True user = jsonuser break gwroot = self.db.get_config('gw_root') if gwroot == None: raise Exception('unable to rip gonewild albums: gw_root is null') userroot = path.join(gwroot, user) # Check if we can actually rip this user if not found or not path.exists(userroot): return { 'error' : 'unable to rip user (not archived)' } # Create subdirs ImageUtils.create_subdirectories(path.join(savedir, 'thumbs')) # Copy images to /rips/, get values that need to be inserted into db (insertmany) insertmany = [] already_got = [] filesize = 0 for root, subdirs, files in walk(userroot): if root.endswith('thumbs'): continue for filename in sorted(files): f = path.join(root, filename) n = filename if not root.endswith(userroot): # It's a subidr, save the file accordingly n = '%s_%s' % (root[root.rfind('/')+1:], filename) # Avoid duplicates no_post = n[n.rfind('_')+1:] if no_post in already_got: continue already_got.append(no_post) n = '%03d_%s' % (len(insertmany) + 1, n) saveas = path.join(savedir, n) # Copy & get size try: copy2(f, saveas) (width, height) = ImageUtils.get_dimensions(saveas) except Exception, e: # image can't be parsed, probably corrupt. move on. continue # Create thumbnail tsaveas = path.join(savedir, 'thumbs', n) try: (tsaveas, twidth, theight) = ImageUtils.create_thumbnail(saveas, tsaveas) except Exception, e: # Failed to create thumb tsaveas = '/'.join(['ui', 'images', 'nothumb.png']) twidth = theight = 160 filesize += path.getsize(saveas) # Add to list of values to insert into DB insertmany.append( [ self.album_id, # album_id, currently None len(insertmany) + 1, # i_index '', # url TODO 1, # valid None, # error SiteBase.get_type(saveas), # type n, # image_name width, # img width height, # img height path.getsize(saveas), # filesize path.basename(tsaveas), # thumb_name twidth, # thumb width theight, # thumb height None # metadata ] )
if path.exists(newimage): self.debug('new image already exists: %s' % newimage) return ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs')) copy2(oldpath, newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception, e: self.debug('failed to create thumbnail: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') (post, comment, imgid) = self.get_post_comment_id(oldimage) url = 'http://i.imgur.com/%s' % imgid dims = ImageUtils.get_dimensions(newimage) size = path.getsize(newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception, e: self.debug('add_existing_image: create_thumbnail failed: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') try: self.add_image(newimage, user, url, dims[0], dims[1], size, thumbnail, 'image', album_id, post, comment) except Exception, e: self.debug('add_existing_image: failed: %s' % str(e)) return if subdir == '' and album_id == -1: # Not an album
# Download URL try: self.debug('%s: process_url: downloading #%d %s' % (child.author, media_index + 1, media)) ImageUtils.httpy.download(media, saveas) if path.getsize(saveas) == 503: raise Exception('503b = removed') except Exception, e: self.debug( '%s: process_url: failed to download #%d: %s, moving on' % (child.author, media_index + 1, str(e))) continue # Get media information (width, height, size) try: (width, height) = ImageUtils.get_dimensions(saveas) except Exception, e: # If we cannot process the media file, skip it! self.debug('%s: process_url: #%d %s' % (child.author, media_index + 1, str(e))) continue size = path.getsize(saveas) # Create thumbnail savethumbas = path.join(working_dir, 'thumbs', fname) try: savethumbas = ImageUtils.create_thumbnail(saveas, savethumbas) except Exception, e: savethumbas = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') self.debug(
print "THREAD: %s: failed to download %s to %s: %s\n%s" % ( url["path"], url["url"], saveas, str(e), str(format_exc()), ) result["error"] = "failed to download %s to %s: %s\n%s" % (url["url"], saveas, str(e), str(format_exc())) self.results.append(result) self.current_threads.pop() return # Save image info result["filesize"] = path.getsize(saveas) try: (result["width"], result["height"]) = ImageUtils.get_dimensions(saveas) except Exception, e: # This fails if we can't identify the image file. Consider it errored print "THREAD: %s: failed to identify image file %s from %s: %s\n%s" % ( url["path"], saveas, url["url"], str(e), format_exc(), ) result["error"] = "failed to identify image file %s from %s: %s\n%s" % ( saveas, url["url"], str(e), format_exc(), )
} ImageUtils.httpy.download(media, saveas, headers=headers) if path.getsize(saveas) == 503: raise Exception('503b = removed') except Exception, e: self.debug('%s: process_url: failed to download #%d: %s, moving on' % (child.author, media_index + 1, str(e))) continue # Get media information (width, height, thumbsaveas) if media_type == 'audio': # Audio files don't have width/height/thumbnail width = height = 0 savethumbas = path.join(ImageUtils.get_root(), 'images', 'audio.png') else: try: (width, height) = ImageUtils.get_dimensions(saveas) except Exception, e: # If we cannot process the media file, skip it! self.debug('%s: process_url: #%d %s' % (child.author, media_index + 1, str(e))) continue # Create thumbnail if needed if self.db.get_config('save_thumbnails', 'true') == 'false': savethumbas = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') else: savethumbas = path.join(working_dir, 'thumbs', fname) try: savethumbas = ImageUtils.create_thumbnail(saveas, savethumbas) except Exception, e: savethumbas = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') self.debug('%s: process_url: failed to create thumb #%d: %s, using default' % (child.author, media_index + 1, str(e)))
def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1): if 'tumblr' in oldpath: # Can't properly handle tumblr links self.debug('cannot properly handle tumblr links; trying anyway') #return if subdir == '' and album_id == -1: self.debug('adding image: %s' % oldpath) # Ensure image is an actual image try: dims = ImageUtils.get_dimensions(oldpath) except: self.debug('failed to load image: %s, skipping' % oldpath) return newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage) newimage = newimage.replace('.jpeg.jpg', '.jpg') thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage) thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg') if path.exists(newimage): self.debug('new image already exists: %s' % newimage) return ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs')) copy2(oldpath, newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception as e: self.debug('failed to create thumbnail: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') (post, comment, imgid) = self.get_post_comment_id(oldimage) url = 'http://i.imgur.com/%s' % imgid dims = ImageUtils.get_dimensions(newimage) size = path.getsize(newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception as e: self.debug('add_existing_image: create_thumbnail failed: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') try: self.add_image(newimage, user, url, dims[0], dims[1], size, thumbnail, 'image', album_id, post, comment) except Exception as e: self.debug('add_existing_image: failed: %s' % str(e)) return if subdir == '' and album_id == -1: # Not an album # Add post p = Post() p.id = post p.author = user if comment == None: p.url = url p.created = path.getctime(oldpath) p.subreddit = '' p.title = '' try: self.add_post(p, legacy=1) except Exception as e: self.debug('add_existing_image: create post failed: %s' % str(e)) # Add comment if comment != None: c = Comment() c.id = comment c.post_id = post c.author = user if comment != None: c.body = url p.created = path.getctime(oldpath) try: self.add_comment(c, legacy=1) except Exception as e: self.debug('add_existing_image: create comment failed: %s' % str(e))
result['image_name'] = path.basename(saveas) # Attempt to dowload image at URL try: self.httpy.download(url['url'], saveas) except Exception, e: print 'THREAD: %s: failed to download %s to %s: %s\n%s' % (url['path'], url['url'], saveas, str(e), str(format_exc())) result['error'] = 'failed to download %s to %s: %s\n%s' % (url['url'], saveas, str(e), str(format_exc())) self.to_remove.append( (url['album_id'], url['i_index'] ) ) self.results.append(result) self.current_threads.pop() return # Save image info result['filesize'] = path.getsize(saveas) try: (result['width'], result['height']) = ImageUtils.get_dimensions(saveas) except Exception, e: # This fails if we can't identify the image file. Consider it errored print 'THREAD: %s: failed to identify image file %s from %s: %s\n%s' % (url['path'], saveas, url['url'], str(e), format_exc()) result['error'] = 'failed to identify image file %s from %s: %s\n%s' % (saveas, url['url'], str(e), format_exc()) self.to_remove.append( (url['album_id'], url['i_index'] ) ) self.results.append(result) self.current_threads.pop() return # Get thumbnail tsaveas = path.join(dirname, 'thumbs', url['saveas']) try: (tsaveas, result['t_width'], result['t_height']) = ImageUtils.create_thumbnail(saveas, tsaveas) except Exception, e: # Failed to create thumbnail, use default