Пример #1
0
 def next_image(self):
     """shout next random image from public to redis"""
     image = self.random_image(folder="public")
     if image:
         LOGGER.info("shouting next image: %s" % (image))
         self.redis_shout(self.__redis_opt["image_pubsub"], image)
         print("shouting next image: %s" % (image))
Пример #2
0
 def jinja_static_file(self, name, folder='public', full=False):
     '''returns jinja filepath for static filenames'''
     if folder in self.get_contentsub(full=False):
         fullfile = self.path_join(folder, name)
         if path.exists(fullfile):
             LOGGER.info('file exists')
             return fullfile if full else '/'.join(fullfile.split('/')[-3:])
Пример #3
0
 def check(self):
     hashmap = dict()
     match = 0
     subfolders = self.__files.get_contentsub(full=True)
     for folder in subfolders:
         for filename in listdir(subfolders[folder]):
             currentfile = self.__files.path_join(folder, filename)
             if any(currentfile.endswith(x) for x in IMAGEEXTENSIONS):
                 hashobj = self.__hashalg()
                 with open(currentfile, 'rb') as fileobj:
                     for chunk in self.__chunk_reader(fileobj):
                         hashobj.update(chunk)
                     fileID = (hashobj.digest(), path.getsize(currentfile))
                     duplicate = hashmap.get(fileID, None)
                     if duplicate:
                         match += 1
                         yield {
                             'image': filename,
                             'imgpath': self.__files.jinja_static_file(
                                 filename, folder=folder, full=False
                                 )
                             }
                     else:
                         hashmap[fileID] = currentfile
     LOGGER.info('found %i duplicate files' %(match))
Пример #4
0
 def __init__(self):
     super().__init__()
     if not self.__net:
         self.__net = Net()
     if not self._users:
         self._users = SOUPUSERS
     LOGGER.info('new suppenkasper instance')
Пример #5
0
 def browser_shout(self, channel):
     pubsub = self.__rdb.pubsub()
     pubsub.subscribe(channel)
     for event in pubsub.listen():
         LOGGER.info("shouting to browser channel:%s message:%s" % (channel, event["data"]))
         if event["type"] == "message":
             strdata = "data: %s\r\n\r\n" % (event["data"].replace("\n", "<br />"))
             yield strdata.encode("UTF-8")
Пример #6
0
 def __init__(self, interval, function, *args, **kwargs):
     super().__init__()
     self._interval = interval
     self.function = function
     self.args = args
     self.kwargs = kwargs
     self.is_running = False
     LOGGER.info('new cycle created %s (%s seconds)' %(self.function.__name__, self._interval))
Пример #7
0
 def __readin():
     """reloads db"""
     self.__rdb.delete(rdbfield)
     for image in self.__files.find_images(folder=folder):
         self.__rdb.rpush(rdbfield, image)
         result.append(image)
     LOGGER.info("rebuilt redis image cache for %s" % (rdbfield))
     return result
Пример #8
0
def internal_error(error):
    '''500'''
    LOGGER.error(error)
    flash(error)
    return render_template('main.html',
        title='500',
        error='This is weird!',
        ), 500
Пример #9
0
def not_found(error):
    '''404'''
    LOGGER.error(error)
    flash(error)
    return render_template('main.html',
        title='404',
        error='I checked twice!',
        ), 404
Пример #10
0
 def _run(self):
     if self.is_running:
         LOGGER.info('cycle tick %s (%s seconds)' %(self.function.__name__, self._interval))
         p = Process(target=self.function, args=self.args, kwargs=self.kwargs)
         p.start()
         p.join()
         Timer(self._interval, self.start).start()
     self.is_running = False
Пример #11
0
    def flush_all(self):
        rdbfields = list()
        for folder in self.__files.get_contentsub(full=False):
            rdbfields.append("%s:%s" % (self.__redis_opt["image_prefix"], folder))
        rdbfields.append("%s:feed" % (self.__redis_opt["status_prefix"]))

        for entry in rdbfields:
            self.__rdb.delete(entry)
            LOGGER.info("flushed data for %s" % (entry))
Пример #12
0
    def redis_ping(self):
        if self.__rdb:
            try:
                LOGGER.info("redis ping")
                return self.__rdb.ping()
            except RedisConnectionError as ex:
                LOGGER.error("could not ping redis: %s" % (ex))

        return False
Пример #13
0
 def kasper(self):
     load = list()
     for user in self._users:
         LOGGER.info('kasper %s' %(user))
         for image in self._usercrawl(user):
             if image.split('/')[-1] not in RDB.get_all_images():
                 load.append(image)
                 yield user, image
     tpool(self.__net.download_image, load)
Пример #14
0
def stream_channel(channel=None):
    if channel:
        if channel in [APP.config['REDIS_OPT'][psc] for psc in APP.config['REDIS_OPT'] if psc.endswith('_pubsub')]:
            LOGGER.info('stream for %s requested' %(channel))
            return Response(
                stream_with_context(RDB.browser_shout(APP.config['REDIS_OPT']['%s_pubsub' %(channel)])),
                direct_passthrough=True,
                mimetype='text/event-stream'
                )
Пример #15
0
 def _usercrawl(self, user, loops=SOUPPAGES):
     since = ''
     for loop in range(loops):
         url = 'http://{user}.soup.io/{since}'.format(user=user, since=since)
         self.__scrape = self.__net.url_scrape(url, split=True)
         since = self.__nextsince()
         LOGGER.info('finished page %d/%d for %s' %(loop+1, loops, user))
         for image in self.__pageimages():
             yield image
Пример #16
0
 def locate_image(self, name):
     """locates images"""
     folder = self.__dblocate_image(name)
     if folder:
         LOGGER.info("found requested image %s in folder %s" % (name, folder))
         image = self.__files.jinja_static_file(name, folder=folder)
         if image:
             return image
     LOGGER.info("requested image %s not found" % (name))
     return self.__files.jinja_nullimg()
Пример #17
0
 def __readin():
     """reloads db"""
     self.__rdb.delete(rdbfield)
     result = self.__net.url_scrape(self.__redis_opt["status_url"])
     if result:
         self.__rdb.set(rdbfield, result)
         self.__rdb.expire(rdbfield, self.__redis_opt["status_expire"])
         LOGGER.info(
             "rebuilt redis status cache for %s, again after %i seconds"
             % (rdbfield, self.__redis_opt["status_expire"])
         )
         return loads(result)
Пример #18
0
def index():
    '''homepage'''
    sidebar = {
        'status': RDB.get_status(),
        'imagestats': RDB.get_imagestats(),
        'tagline': next(TAGLINES),
        }
    LOGGER.info('index requested')
    return render_template('main.html',
        title='fnordpad',
        image=RDB.random_image(folder='public'),
        sidebar=sidebar,
        )
Пример #19
0
    def delete(self):
        for dup in self.check():
            fullpath = self.__files.jinja_static_file(
                dup['image'],
                folder=dup['imgpath'].split('/')[-2],
                full=True
                )
            if path.exists(fullpath):
                remove(fullpath)
                print('deleted %s' %(fullpath)) # as explicit warning to the error.log
                LOGGER.info('deleted duplicate %s' %(dup['image']))
                yield {'image': dup['image'], 'imgpath': dup['imgpath']}

        LOGGER.info('no duplicate files deleted')
Пример #20
0
 def __init__(self, redis_opt):
     super().__init__()
     if not self.__rdb:
         self.__rdb = StrictRedis(
             host=redis_opt["host"],
             port=redis_opt["port"],
             db=redis_opt["db"],
             decode_responses=redis_opt["decode_responses"],
         )
         self.__redis_opt = redis_opt
         LOGGER.info("created new redis connection")
     if not self.__files:
         self.__files = Files()
     if not self.__net:
         self.__net = Net()
     if not self.__proc:
         self.__proc = Cycle(redis_opt["image_timeout"], self.next_image).start()
         pass
Пример #21
0
 def download_image(self, url):
     try:
         LOGGER.info('downloading %s' %(url))
         filename = url.split('/')[-1]
         r = rget(url, stream=True)
         if r.status_code == 200:
             with open(path.join(CONTENTSUB['unsorted'], filename), 'wb') as f:
                 for chunk in r.iter_content(1024):
                     f.write(chunk)
             LOGGER.info('done: %s' %(filename))
     except Exception as ex:
         LOGGER.error('download error %s %s' %(url, ex))
Пример #22
0
 def url_scrape(self, url, split=False):
     try:
         LOGGER.info('scraping %s' %(url))
         return rget(url).text.split('\n') if split else rget(url).text
     except Exception as ex:
         LOGGER.error('scrape error %s %s' %(url, ex))
Пример #23
0
def tpool(function, data):
    LOGGER.info('new tpool %s - %s' %(function.__name__, data))
    pool = Pool()
    pool.map(function, data)
    pool.close()
    pool.join()
Пример #24
0
 def start(self):
     self.is_running = True
     LOGGER.info('cycle started %s (%s seconds)' %(self.function.__name__, self._interval))
     self._run()
Пример #25
0
 def errorsplash(brain=None):
     '''help - no brain found'''
     LOGGER.info('request was: %s' %(brain))
     return views.redis_error('no brain found: %s' %(DB_ERRMSG))
Пример #26
0
 def file_rename(self, source, target):
     if path.exists(source):
         LOGGER.info('moved %s to %s' %(source, target))
         rename(source, target)
Пример #27
0
 def stop(self):
     self.is_running = False
     LOGGER.info('cycle stopped %s (%s seconds)' %(self.function.__name__, self._interval))
Пример #28
0
 def __init__(self):
     super().__init__()
     LOGGER.info('new net instance')
Пример #29
0
 def __init__(self):
     super().__init__()
     if not self.__files:
         self.__files = Files()
     LOGGER.info('new duplicates instance')
Пример #30
0
 def redis_shout(self, channel, message):
     LOGGER.info("shouting to redis channel:%s message:%s" % (channel, message))
     self.__rdb.publish(channel, message)
     return message