def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1):
if 'tumblr' in oldpath:
# Can't properly handle tumblr links
self.debug('cannot properly handle tumblr links; trying anyway')
#return
if subdir == '' and album_id == -1:
self.debug('adding image: %s' % oldpath)
# Ensure image is an actual image
try:
dims = ImageUtils.get_dimensions(oldpath)
except:
self.debug('failed to load image: %s, skipping' % oldpath)
return
newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage)
newimage = newimage.replace('.jpeg.jpg', '.jpg')
thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage)
thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg')
if path.exists(newimage):
self.debug('new image already exists: %s' % newimage)
return
ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs'))
copy2(oldpath, newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception, e:
self.debug('failed to create thumbnail: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
def setup_loggers_for_user(self, user): # Create directories if needed user_dir = path.join(ImageUtils.get_root(), 'content', user) ImageUtils.create_subdirectories(user_dir) # Setup logger self.logger = open(path.join(user_dir, 'history.log'), 'a') self.db.logger = self.logger ImageUtils.logger = self.logger self.reddit.logger = self.logger
def setup_loggers_for_user(self, user):
# Create directories if needed
user_dir = path.join(ImageUtils.get_root(), 'content', user)
ImageUtils.create_subdirectories(user_dir)
# Setup logger
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none': self.logger = open(devnull, 'w')
elif log_level == 'user': self.logger = open(path.join(user_dir, 'history.log'), 'a')
elif log_level == 'global': self.logger = self.root_log
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
def setup_loggers_for_user(self, user):
# Create directories if needed
user_dir = path.join(ImageUtils.get_root(), 'content', user)
ImageUtils.create_subdirectories(user_dir)
# Setup logger
log_level = self.db.get_config('log_level', default='user')
if log_level == 'none': self.logger = open(devnull, 'w')
elif log_level == 'user':
self.logger = open(path.join(user_dir, 'history.log'), 'a')
elif log_level == 'global':
self.logger = self.root_log
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
def start(self):
stale_count = self.db.count('urls', 'pending != 0')
if stale_count > 0:
print 'MAIN: found %d stale (interrupted) URLs, marking as non-pending...' % stale_count
self.db.update('urls', 'pending = 0')
self.db.commit()
print 'MAIN: starting infinite loop...'
already_printed_sleep_msg = False
while True:
sleep(0.1)
while len(self.results) > 0:
# self.results is the list of downloaded medias to be added to the DB
result = self.results.pop()
self.handle_result(result)
# Remove recently-completed rips
while len(self.to_remove) > 0:
(albumid, iindex) = self.to_remove.pop()
self.db.delete('urls', 'album_id = ? and i_index = ?', [ albumid, iindex ] )
self.db.commit()
try:
# Get next URL to retrieve
url = self.get_next_url()
except Exception, e:
if str(e) == 'no URLs found':
if not already_printed_sleep_msg:
already_printed_sleep_msg = True
print 'MAIN: no urls to get, sleeping 500ms'
sleep(0.5)
else:
print 'MAIN: get_next_url(): Exception: %s:\n%s' % (str(e), format_exc())
continue
# We have a URL to download & add to DB (url)
already_printed_sleep_msg = False
# Wait for thread count to drop
while len(self.current_threads) >= MAX_THREADS:
sleep(0.1)
self.current_threads.append(None)
# Create new thread to download the media, add to self.results
print 'MAIN: %s #%d: launching handler for: %s' % (url['path'], url['i_index'], url['url'])
# Create subdirs from main thread to avoid race condition
dirname = path.join(ImageUtils.get_root(), 'rips', url['path'], 'thumbs')
ImageUtils.create_subdirectories(dirname)
args = (url,)
t = Thread(target=self.retrieve_result_from_url, args=args)
t.start()
def start(self):
print "MAIN: starting infinite loop..."
already_printed_sleep_msg = False
while True:
sleep(0.1)
if len(self.results) > 0:
# self.results is the list of downloaded medias to be added to the DB
result = self.results.pop()
self.handle_result(result)
try:
# Get next URL to retrieve
url = self.get_next_url()
except Exception, e:
if str(e) == "no URLs found":
if not already_printed_sleep_msg:
already_printed_sleep_msg = True
print "MAIN: no urls to get, sleeping 500ms"
sleep(0.5)
else:
print "MAIN: get_next_url(): Exception: %s:\n%s" % (str(e), format_exc())
continue
# We have a URL to download & add to DB (url)
already_printed_sleep_msg = False
# Wait for thread count to drop
while len(self.current_threads) >= MAX_THREADS:
sleep(0.1)
self.current_threads.append(None)
# Create new thread to download the media, add to self.results
print "MAIN: %s #%d: launching handler for: %s" % (url["path"], url["i_index"], url["url"])
# Create subdirs from main thread to avoid race condition
dirname = path.join(ImageUtils.get_root(), "rips", url["path"], "thumbs")
ImageUtils.create_subdirectories(dirname)
args = (url,)
t = Thread(target=self.retrieve_result_from_url, args=args)
t.start()
def poll_user(self, user):
# Create directories if needed
user_dir = path.join(ImageUtils.get_root(), 'content', user)
ImageUtils.create_subdirectories(user_dir)
# Setup logger
self.logger = open(path.join(user_dir, 'history.log'), 'a')
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
since_id = self.db.get_last_since_id(user)
# Get posts/comments for user
self.debug('%s: poll_user: since "%s"' % (user, since_id))
try:
children = self.reddit.get_user(user, since=since_id)
except Exception, e:
if '404: Not Found' in str(e):
# User is deleted, mark it as such
self.debug('%s: poll_user: user is 404, marking as deleted' % user)
self.db.mark_as_deleted(user)
return
self.debug('%s: poll_user: error %s' % (user, str(e)))
return
def poll_user(self, user):
# Create directories if needed
user_dir = path.join(ImageUtils.get_root(), 'content', user)
ImageUtils.create_subdirectories(user_dir)
# Setup logger
self.logger = open(path.join(user_dir, 'history.log'), 'a')
self.db.logger = self.logger
ImageUtils.logger = self.logger
self.reddit.logger = self.logger
since_id = self.db.get_last_since_id(user)
# Get posts/comments for user
self.debug('%s: poll_user: since "%s"' % (user, since_id))
try:
children = self.reddit.get_user(user, since=since_id)
except Exception, e:
if '404: Not Found' in str(e):
# User is deleted, mark it as such
self.debug('%s: poll_user: user is 404, marking as deleted' %
user)
self.db.mark_as_deleted(user)
return
self.debug('%s: poll_user: error %s' % (user, str(e)))
return
def start(self):
'''
Overriding SiteBase's start() method for unique ripping logic
'''
# We need a lot of libraries
from ImageUtils import ImageUtils
from calendar import timegm
from shutil import copy2, rmtree
from time import gmtime
from os import path, walk, environ, getcwd
from json import loads
savedir = path.join('rips', self.path)
if getcwd().endswith('py'):
savedir = path.join('..', savedir)
if self.album_exists:
# Don't re-rip an album. Return info about existing album.
return {
'warning' : 'album already exists',
'album_id' : self.album_id,
'album' : self.album_name,
'url' : self.url,
'host' : self.get_host(),
'path' : self.path,
'count' : self.db.count('medias', 'album_id = ?', [self.album_id]),
'pending' : self.db.count('urls', 'album_id = ?', [self.album_id])
}
user = self.url.split(':')[-1]
# Search for username (with proper case) on site
gwapi = self.db.get_config('gw_api')
if gwapi == None:
raise Exception('unable to rip gonewild albums: gw_api is null')
r = self.httpy.get('%s?method=search_user&user=%s' % (gwapi, user))
json = loads(r)
found = False
for jsonuser in json['users']:
if jsonuser.lower() == user.lower():
found = True
user = jsonuser
break
gwroot = self.db.get_config('gw_root')
if gwroot == None:
raise Exception('unable to rip gonewild albums: gw_root is null')
userroot = path.join(gwroot, user)
# Check if we can actually rip this user
if not found or not path.exists(userroot):
return {
'error' : 'unable to rip user (not archived)'
}
# Create subdirs
ImageUtils.create_subdirectories(path.join(savedir, 'thumbs'))
# Copy images to /rips/, get values that need to be inserted into db (insertmany)
insertmany = []
already_got = []
filesize = 0
for root, subdirs, files in walk(userroot):
if root.endswith('thumbs'): continue
for filename in sorted(files):
f = path.join(root, filename)
n = filename
if not root.endswith(userroot):
# It's a subidr, save the file accordingly
n = '%s_%s' % (root[root.rfind('/')+1:], filename)
# Avoid duplicates
no_post = n[n.rfind('_')+1:]
if no_post in already_got: continue
already_got.append(no_post)
n = '%03d_%s' % (len(insertmany) + 1, n)
saveas = path.join(savedir, n)
# Copy & get size
try:
copy2(f, saveas)
(width, height) = ImageUtils.get_dimensions(saveas)
except Exception, e:
# image can't be parsed, probably corrupt. move on.
continue
# Create thumbnail
tsaveas = path.join(savedir, 'thumbs', n)
try:
(tsaveas, twidth, theight) = ImageUtils.create_thumbnail(saveas, tsaveas)
except Exception, e:
# Failed to create thumb
tsaveas = '/'.join(['ui', 'images', 'nothumb.png'])
twidth = theight = 160
filesize += path.getsize(saveas)
# Add to list of values to insert into DB
insertmany.append( [
self.album_id, # album_id, currently None
len(insertmany) + 1, # i_index
'', # url TODO
1, # valid
None, # error
SiteBase.get_type(saveas), # type
n, # image_name
width, # img width
height, # img height
path.getsize(saveas), # filesize
path.basename(tsaveas), # thumb_name
twidth, # thumb width
theight, # thumb height
None # metadata
] )
if albumname != None:
# Album!
albumname = '%s-%s' % (base_fname, albumname)
working_dir = path.join(working_dir, albumname)
#self.debug('%s: process_url: adding album to database' % child.author)
album_id = self.db.add_album(
working_dir,
child.author,
url,
postid,
commid,
)
else:
album_id = None
ImageUtils.create_subdirectories(path.join(working_dir, 'thumbs'))
for media_index, media in enumerate(medias):
# Construct save path: /user/post[-comment]-index-filename
fname = ImageUtils.get_filename_from_url(media)
fname = '%s-%02d-%s' % (base_fname, media_index, fname)
saveas = path.join(working_dir, fname)
# Download URL
try:
self.debug('%s: process_url: downloading #%d %s' %
(child.author, media_index + 1, media))
ImageUtils.httpy.download(media, saveas)
if path.getsize(saveas) == 503:
raise Exception('503b = removed')
except Exception, e:
# Album!
albumname = '%s-%s' % (base_fname, albumname)
working_dir = path.join(working_dir, albumname)
#self.debug('%s: process_url: adding album to database' % child.author)
album_id = self.db.add_album(
working_dir,
child.author,
url,
postid,
commid,
)
else:
album_id = None
if self.db.get_config('save_thumbnails', default='true') == 'true':
ImageUtils.create_subdirectories(path.join(working_dir, 'thumbs'))
else:
ImageUtils.create_subdirectories(working_dir)
for media_index, media in enumerate(medias):
# Construct save path: /user/post[-comment]-index-filename
fname = ImageUtils.get_filename_from_url(media, media_type)
fname = '%s-%02d-%s' % (base_fname, media_index, fname)
saveas = path.join(working_dir, fname)
# Download URL
try:
self.debug('%s: process_url: downloading #%d %s' % (child.author, media_index + 1, media))
headers = {
'Referer' : url
}
def add_existing_image(self, user, oldimage, oldpath, subdir='', album_id=-1):
if 'tumblr' in oldpath:
# Can't properly handle tumblr links
self.debug('cannot properly handle tumblr links; trying anyway')
#return
if subdir == '' and album_id == -1:
self.debug('adding image: %s' % oldpath)
# Ensure image is an actual image
try:
dims = ImageUtils.get_dimensions(oldpath)
except:
self.debug('failed to load image: %s, skipping' % oldpath)
return
newimage = path.join(ImageUtils.get_root(), 'content', user, subdir, oldimage)
newimage = newimage.replace('.jpeg.jpg', '.jpg')
thumbnail = path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs', oldimage)
thumbnail = thumbnail.replace('.jpeg.jpg', '.jpg')
if path.exists(newimage):
self.debug('new image already exists: %s' % newimage)
return
ImageUtils.create_subdirectories(path.join(ImageUtils.get_root(), 'content', user, subdir, 'thumbs'))
copy2(oldpath, newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception as e:
self.debug('failed to create thumbnail: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
(post, comment, imgid) = self.get_post_comment_id(oldimage)
url = 'http://i.imgur.com/%s' % imgid
dims = ImageUtils.get_dimensions(newimage)
size = path.getsize(newimage)
try:
ImageUtils.create_thumbnail(newimage, thumbnail)
except Exception as e:
self.debug('add_existing_image: create_thumbnail failed: %s' % str(e))
thumbnail = path.join(ImageUtils.get_root(), 'images', 'nothumb.png')
try:
self.add_image(newimage, user, url,
dims[0], dims[1], size, thumbnail, 'image',
album_id, post, comment)
except Exception as e:
self.debug('add_existing_image: failed: %s' % str(e))
return
if subdir == '' and album_id == -1: # Not an album
# Add post
p = Post()
p.id = post
p.author = user
if comment == None: p.url = url
p.created = path.getctime(oldpath)
p.subreddit = ''
p.title = ''
try:
self.add_post(p, legacy=1)
except Exception as e:
self.debug('add_existing_image: create post failed: %s' % str(e))
# Add comment
if comment != None:
c = Comment()
c.id = comment
c.post_id = post
c.author = user
if comment != None: c.body = url
p.created = path.getctime(oldpath)
try:
self.add_comment(c, legacy=1)
except Exception as e:
self.debug('add_existing_image: create comment failed: %s' % str(e))
