def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1): if 'tumblr' in oldpath: # Can't properly handle tumblr links self.debug('cannot properly handle tumblr links; trying anyway') #return if subdir == '' and album_id == -1: self.debug('adding image: %s' % oldpath) # Ensure image is an actual image try: dims = ImageUtils.get_dimensions(oldpath) except: self.debug('failed to load image: %s, skipping' % oldpath) return newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage) newimage = newimage.replace('.jpeg.jpg', '.jpg') thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage) thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg') if path.exists(newimage): self.debug('new image already exists: %s' % newimage) return ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs')) copy2(oldpath, newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception, e: self.debug('failed to create thumbnail: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
def setup_loggers_for_user(self, user): # Create directories if needed user_dir = path.join(ImageUtils.get_root(), 'content', user) ImageUtils.create_subdirectories(user_dir) # Setup logger self.logger = open(path.join(user_dir, 'history.log'), 'a') self.db.logger = self.logger ImageUtils.logger = self.logger self.reddit.logger = self.logger
def setup_loggers_for_user(self, user): # Create directories if needed user_dir = path.join(ImageUtils.get_root(), 'content', user) ImageUtils.create_subdirectories(user_dir) # Setup logger log_level = self.db.get_config('log_level', default='user') if log_level == 'none': self.logger = open(devnull, 'w') elif log_level == 'user': self.logger = open(path.join(user_dir, 'history.log'), 'a') elif log_level == 'global': self.logger = self.root_log self.db.logger = self.logger ImageUtils.logger = self.logger self.reddit.logger = self.logger
def start(self): stale_count = self.db.count('urls', 'pending != 0') if stale_count > 0: print 'MAIN: found %d stale (interrupted) URLs, marking as non-pending...' % stale_count self.db.update('urls', 'pending = 0') self.db.commit() print 'MAIN: starting infinite loop...' already_printed_sleep_msg = False while True: sleep(0.1) while len(self.results) > 0: # self.results is the list of downloaded medias to be added to the DB result = self.results.pop() self.handle_result(result) # Remove recently-completed rips while len(self.to_remove) > 0: (albumid, iindex) = self.to_remove.pop() self.db.delete('urls', 'album_id = ? and i_index = ?', [ albumid, iindex ] ) self.db.commit() try: # Get next URL to retrieve url = self.get_next_url() except Exception, e: if str(e) == 'no URLs found': if not already_printed_sleep_msg: already_printed_sleep_msg = True print 'MAIN: no urls to get, sleeping 500ms' sleep(0.5) else: print 'MAIN: get_next_url(): Exception: %s:\n%s' % (str(e), format_exc()) continue # We have a URL to download & add to DB (url) already_printed_sleep_msg = False # Wait for thread count to drop while len(self.current_threads) >= MAX_THREADS: sleep(0.1) self.current_threads.append(None) # Create new thread to download the media, add to self.results print 'MAIN: %s #%d: launching handler for: %s' % (url['path'], url['i_index'], url['url']) # Create subdirs from main thread to avoid race condition dirname = path.join(ImageUtils.get_root(), 'rips', url['path'], 'thumbs') ImageUtils.create_subdirectories(dirname) args = (url,) t = Thread(target=self.retrieve_result_from_url, args=args) t.start()
def start(self): print "MAIN: starting infinite loop..." already_printed_sleep_msg = False while True: sleep(0.1) if len(self.results) > 0: # self.results is the list of downloaded medias to be added to the DB result = self.results.pop() self.handle_result(result) try: # Get next URL to retrieve url = self.get_next_url() except Exception, e: if str(e) == "no URLs found": if not already_printed_sleep_msg: already_printed_sleep_msg = True print "MAIN: no urls to get, sleeping 500ms" sleep(0.5) else: print "MAIN: get_next_url(): Exception: %s:\n%s" % (str(e), format_exc()) continue # We have a URL to download & add to DB (url) already_printed_sleep_msg = False # Wait for thread count to drop while len(self.current_threads) >= MAX_THREADS: sleep(0.1) self.current_threads.append(None) # Create new thread to download the media, add to self.results print "MAIN: %s #%d: launching handler for: %s" % (url["path"], url["i_index"], url["url"]) # Create subdirs from main thread to avoid race condition dirname = path.join(ImageUtils.get_root(), "rips", url["path"], "thumbs") ImageUtils.create_subdirectories(dirname) args = (url,) t = Thread(target=self.retrieve_result_from_url, args=args) t.start()
def poll_user(self, user): # Create directories if needed user_dir = path.join(ImageUtils.get_root(), 'content', user) ImageUtils.create_subdirectories(user_dir) # Setup logger self.logger = open(path.join(user_dir, 'history.log'), 'a') self.db.logger = self.logger ImageUtils.logger = self.logger self.reddit.logger = self.logger since_id = self.db.get_last_since_id(user) # Get posts/comments for user self.debug('%s: poll_user: since "%s"' % (user, since_id)) try: children = self.reddit.get_user(user, since=since_id) except Exception, e: if '404: Not Found' in str(e): # User is deleted, mark it as such self.debug('%s: poll_user: user is 404, marking as deleted' % user) self.db.mark_as_deleted(user) return self.debug('%s: poll_user: error %s' % (user, str(e))) return
def start(self): ''' Overriding SiteBase's start() method for unique ripping logic ''' # We need a lot of libraries from ImageUtils import ImageUtils from calendar import timegm from shutil import copy2, rmtree from time import gmtime from os import path, walk, environ, getcwd from json import loads savedir = path.join('rips', self.path) if getcwd().endswith('py'): savedir = path.join('..', savedir) if self.album_exists: # Don't re-rip an album. Return info about existing album. return { 'warning' : 'album already exists', 'album_id' : self.album_id, 'album' : self.album_name, 'url' : self.url, 'host' : self.get_host(), 'path' : self.path, 'count' : self.db.count('medias', 'album_id = ?', [self.album_id]), 'pending' : self.db.count('urls', 'album_id = ?', [self.album_id]) } user = self.url.split(':')[-1] # Search for username (with proper case) on site gwapi = self.db.get_config('gw_api') if gwapi == None: raise Exception('unable to rip gonewild albums: gw_api is null') r = self.httpy.get('%s?method=search_user&user=%s' % (gwapi, user)) json = loads(r) found = False for jsonuser in json['users']: if jsonuser.lower() == user.lower(): found = True user = jsonuser break gwroot = self.db.get_config('gw_root') if gwroot == None: raise Exception('unable to rip gonewild albums: gw_root is null') userroot = path.join(gwroot, user) # Check if we can actually rip this user if not found or not path.exists(userroot): return { 'error' : 'unable to rip user (not archived)' } # Create subdirs ImageUtils.create_subdirectories(path.join(savedir, 'thumbs')) # Copy images to /rips/, get values that need to be inserted into db (insertmany) insertmany = [] already_got = [] filesize = 0 for root, subdirs, files in walk(userroot): if root.endswith('thumbs'): continue for filename in sorted(files): f = path.join(root, filename) n = filename if not root.endswith(userroot): # It's a subidr, save the file accordingly n = '%s_%s' % (root[root.rfind('/')+1:], filename) # Avoid duplicates no_post = n[n.rfind('_')+1:] if no_post in already_got: continue already_got.append(no_post) n = '%03d_%s' % (len(insertmany) + 1, n) saveas = path.join(savedir, n) # Copy & get size try: copy2(f, saveas) (width, height) = ImageUtils.get_dimensions(saveas) except Exception, e: # image can't be parsed, probably corrupt. move on. continue # Create thumbnail tsaveas = path.join(savedir, 'thumbs', n) try: (tsaveas, twidth, theight) = ImageUtils.create_thumbnail(saveas, tsaveas) except Exception, e: # Failed to create thumb tsaveas = '/'.join(['ui', 'images', 'nothumb.png']) twidth = theight = 160 filesize += path.getsize(saveas) # Add to list of values to insert into DB insertmany.append( [ self.album_id, # album_id, currently None len(insertmany) + 1, # i_index '', # url TODO 1, # valid None, # error SiteBase.get_type(saveas), # type n, # image_name width, # img width height, # img height path.getsize(saveas), # filesize path.basename(tsaveas), # thumb_name twidth, # thumb width theight, # thumb height None # metadata ] )
if albumname != None: # Album! albumname = '%s-%s' % (base_fname, albumname) working_dir = path.join(working_dir, albumname) #self.debug('%s: process_url: adding album to database' % child.author) album_id = self.db.add_album( working_dir, child.author, url, postid, commid, ) else: album_id = None ImageUtils.create_subdirectories(path.join(working_dir, 'thumbs')) for media_index, media in enumerate(medias): # Construct save path: /user/post[-comment]-index-filename fname = ImageUtils.get_filename_from_url(media) fname = '%s-%02d-%s' % (base_fname, media_index, fname) saveas = path.join(working_dir, fname) # Download URL try: self.debug('%s: process_url: downloading #%d %s' % (child.author, media_index + 1, media)) ImageUtils.httpy.download(media, saveas) if path.getsize(saveas) == 503: raise Exception('503b = removed') except Exception, e:
# Album! albumname = '%s-%s' % (base_fname, albumname) working_dir = path.join(working_dir, albumname) #self.debug('%s: process_url: adding album to database' % child.author) album_id = self.db.add_album( working_dir, child.author, url, postid, commid, ) else: album_id = None if self.db.get_config('save_thumbnails', default='true') == 'true': ImageUtils.create_subdirectories(path.join(working_dir, 'thumbs')) else: ImageUtils.create_subdirectories(working_dir) for media_index, media in enumerate(medias): # Construct save path: /user/post[-comment]-index-filename fname = ImageUtils.get_filename_from_url(media, media_type) fname = '%s-%02d-%s' % (base_fname, media_index, fname) saveas = path.join(working_dir, fname) # Download URL try: self.debug('%s: process_url: downloading #%d %s' % (child.author, media_index + 1, media)) headers = { 'Referer' : url }
def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1): if 'tumblr' in oldpath: # Can't properly handle tumblr links self.debug('cannot properly handle tumblr links; trying anyway') #return if subdir == '' and album_id == -1: self.debug('adding image: %s' % oldpath) # Ensure image is an actual image try: dims = ImageUtils.get_dimensions(oldpath) except: self.debug('failed to load image: %s, skipping' % oldpath) return newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage) newimage = newimage.replace('.jpeg.jpg', '.jpg') thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage) thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg') if path.exists(newimage): self.debug('new image already exists: %s' % newimage) return ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs')) copy2(oldpath, newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception as e: self.debug('failed to create thumbnail: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') (post, comment, imgid) = self.get_post_comment_id(oldimage) url = 'http://i.imgur.com/%s' % imgid dims = ImageUtils.get_dimensions(newimage) size = path.getsize(newimage) try: ImageUtils.create_thumbnail(newimage, thumbnail) except Exception as e: self.debug('add_existing_image: create_thumbnail failed: %s' % str(e)) thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png') try: self.add_image(newimage, user, url, dims[0], dims[1], size, thumbnail, 'image', album_id, post, comment) except Exception as e: self.debug('add_existing_image: failed: %s' % str(e)) return if subdir == '' and album_id == -1: # Not an album # Add post p = Post() p.id = post p.author = user if comment == None: p.url = url p.created = path.getctime(oldpath) p.subreddit = '' p.title = '' try: self.add_post(p, legacy=1) except Exception as e: self.debug('add_existing_image: create post failed: %s' % str(e)) # Add comment if comment != None: c = Comment() c.id = comment c.post_id = post c.author = user if comment != None: c.body = url p.created = path.getctime(oldpath) try: self.add_comment(c, legacy=1) except Exception as e: self.debug('add_existing_image: create comment failed: %s' % str(e))