def next_image(self):
"""shout next random image from public to redis"""
image = self.random_image(folder="public")
if image:
LOGGER.info("shouting next image: %s" % (image))
self.redis_shout(self.__redis_opt["image_pubsub"], image)
print("shouting next image: %s" % (image))
def jinja_static_file(self, name, folder='public', full=False):
'''returns jinja filepath for static filenames'''
if folder in self.get_contentsub(full=False):
fullfile = self.path_join(folder, name)
if path.exists(fullfile):
LOGGER.info('file exists')
return fullfile if full else '/'.join(fullfile.split('/')[-3:])
def check(self):
hashmap = dict()
match = 0
subfolders = self.__files.get_contentsub(full=True)
for folder in subfolders:
for filename in listdir(subfolders[folder]):
currentfile = self.__files.path_join(folder, filename)
if any(currentfile.endswith(x) for x in IMAGEEXTENSIONS):
hashobj = self.__hashalg()
with open(currentfile, 'rb') as fileobj:
for chunk in self.__chunk_reader(fileobj):
hashobj.update(chunk)
fileID = (hashobj.digest(), path.getsize(currentfile))
duplicate = hashmap.get(fileID, None)
if duplicate:
match += 1
yield {
'image': filename,
'imgpath': self.__files.jinja_static_file(
filename, folder=folder, full=False
)
}
else:
hashmap[fileID] = currentfile
LOGGER.info('found %i duplicate files' %(match))
def __init__(self):
super().__init__()
if not self.__net:
self.__net = Net()
if not self._users:
self._users = SOUPUSERS
LOGGER.info('new suppenkasper instance')
def browser_shout(self, channel):
pubsub = self.__rdb.pubsub()
pubsub.subscribe(channel)
for event in pubsub.listen():
LOGGER.info("shouting to browser channel:%s message:%s" % (channel, event["data"]))
if event["type"] == "message":
strdata = "data: %s\r\n\r\n" % (event["data"].replace("\n", "<br />"))
yield strdata.encode("UTF-8")
def __init__(self, interval, function, *args, **kwargs):
super().__init__()
self._interval = interval
self.function = function
self.args = args
self.kwargs = kwargs
self.is_running = False
LOGGER.info('new cycle created %s (%s seconds)' %(self.function.__name__, self._interval))
def __readin():
"""reloads db"""
self.__rdb.delete(rdbfield)
for image in self.__files.find_images(folder=folder):
self.__rdb.rpush(rdbfield, image)
result.append(image)
LOGGER.info("rebuilt redis image cache for %s" % (rdbfield))
return result
def internal_error(error):
'''500'''
LOGGER.error(error)
flash(error)
return render_template('main.html',
title='500',
error='This is weird!',
), 500
def not_found(error):
'''404'''
LOGGER.error(error)
flash(error)
return render_template('main.html',
title='404',
error='I checked twice!',
), 404
def _run(self):
if self.is_running:
LOGGER.info('cycle tick %s (%s seconds)' %(self.function.__name__, self._interval))
p = Process(target=self.function, args=self.args, kwargs=self.kwargs)
p.start()
p.join()
Timer(self._interval, self.start).start()
self.is_running = False
def flush_all(self):
rdbfields = list()
for folder in self.__files.get_contentsub(full=False):
rdbfields.append("%s:%s" % (self.__redis_opt["image_prefix"], folder))
rdbfields.append("%s:feed" % (self.__redis_opt["status_prefix"]))
for entry in rdbfields:
self.__rdb.delete(entry)
LOGGER.info("flushed data for %s" % (entry))
def redis_ping(self):
if self.__rdb:
try:
LOGGER.info("redis ping")
return self.__rdb.ping()
except RedisConnectionError as ex:
LOGGER.error("could not ping redis: %s" % (ex))
return False
def kasper(self):
load = list()
for user in self._users:
LOGGER.info('kasper %s' %(user))
for image in self._usercrawl(user):
if image.split('/')[-1] not in RDB.get_all_images():
load.append(image)
yield user, image
tpool(self.__net.download_image, load)
def stream_channel(channel=None):
if channel:
if channel in [APP.config['REDIS_OPT'][psc] for psc in APP.config['REDIS_OPT'] if psc.endswith('_pubsub')]:
LOGGER.info('stream for %s requested' %(channel))
return Response(
stream_with_context(RDB.browser_shout(APP.config['REDIS_OPT']['%s_pubsub' %(channel)])),
direct_passthrough=True,
mimetype='text/event-stream'
)
def _usercrawl(self, user, loops=SOUPPAGES):
since = ''
for loop in range(loops):
url = 'http://{user}.soup.io/{since}'.format(user=user, since=since)
self.__scrape = self.__net.url_scrape(url, split=True)
since = self.__nextsince()
LOGGER.info('finished page %d/%d for %s' %(loop+1, loops, user))
for image in self.__pageimages():
yield image
def locate_image(self, name):
"""locates images"""
folder = self.__dblocate_image(name)
if folder:
LOGGER.info("found requested image %s in folder %s" % (name, folder))
image = self.__files.jinja_static_file(name, folder=folder)
if image:
return image
LOGGER.info("requested image %s not found" % (name))
return self.__files.jinja_nullimg()
def __readin():
"""reloads db"""
self.__rdb.delete(rdbfield)
result = self.__net.url_scrape(self.__redis_opt["status_url"])
if result:
self.__rdb.set(rdbfield, result)
self.__rdb.expire(rdbfield, self.__redis_opt["status_expire"])
LOGGER.info(
"rebuilt redis status cache for %s, again after %i seconds"
% (rdbfield, self.__redis_opt["status_expire"])
)
return loads(result)
def index():
'''homepage'''
sidebar = {
'status': RDB.get_status(),
'imagestats': RDB.get_imagestats(),
'tagline': next(TAGLINES),
}
LOGGER.info('index requested')
return render_template('main.html',
title='fnordpad',
image=RDB.random_image(folder='public'),
sidebar=sidebar,
)
def delete(self):
for dup in self.check():
fullpath = self.__files.jinja_static_file(
dup['image'],
folder=dup['imgpath'].split('/')[-2],
full=True
)
if path.exists(fullpath):
remove(fullpath)
print('deleted %s' %(fullpath)) # as explicit warning to the error.log
LOGGER.info('deleted duplicate %s' %(dup['image']))
yield {'image': dup['image'], 'imgpath': dup['imgpath']}
LOGGER.info('no duplicate files deleted')
def __init__(self, redis_opt):
super().__init__()
if not self.__rdb:
self.__rdb = StrictRedis(
host=redis_opt["host"],
port=redis_opt["port"],
db=redis_opt["db"],
decode_responses=redis_opt["decode_responses"],
)
self.__redis_opt = redis_opt
LOGGER.info("created new redis connection")
if not self.__files:
self.__files = Files()
if not self.__net:
self.__net = Net()
if not self.__proc:
self.__proc = Cycle(redis_opt["image_timeout"], self.next_image).start()
pass
def download_image(self, url):
try:
LOGGER.info('downloading %s' %(url))
filename = url.split('/')[-1]
r = rget(url, stream=True)
if r.status_code == 200:
with open(path.join(CONTENTSUB['unsorted'], filename), 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
LOGGER.info('done: %s' %(filename))
except Exception as ex:
LOGGER.error('download error %s %s' %(url, ex))
def url_scrape(self, url, split=False):
try:
LOGGER.info('scraping %s' %(url))
return rget(url).text.split('\n') if split else rget(url).text
except Exception as ex:
LOGGER.error('scrape error %s %s' %(url, ex))
def tpool(function, data):
LOGGER.info('new tpool %s - %s' %(function.__name__, data))
pool = Pool()
pool.map(function, data)
pool.close()
pool.join()
def start(self):
self.is_running = True
LOGGER.info('cycle started %s (%s seconds)' %(self.function.__name__, self._interval))
self._run()
def errorsplash(brain=None):
'''help - no brain found'''
LOGGER.info('request was: %s' %(brain))
return views.redis_error('no brain found: %s' %(DB_ERRMSG))
def file_rename(self, source, target):
if path.exists(source):
LOGGER.info('moved %s to %s' %(source, target))
rename(source, target)
def stop(self):
self.is_running = False
LOGGER.info('cycle stopped %s (%s seconds)' %(self.function.__name__, self._interval))
def __init__(self):
super().__init__()
LOGGER.info('new net instance')
def __init__(self):
super().__init__()
if not self.__files:
self.__files = Files()
LOGGER.info('new duplicates instance')
def redis_shout(self, channel, message):
LOGGER.info("shouting to redis channel:%s message:%s" % (channel, message))
self.__rdb.publish(channel, message)
return message