def execute(self, options):
log.info("Starting reddrip")
conf = Configuration(options.config)
redis_conn = redis.StrictRedis(
host=conf.glob["redis.host"], port=int(conf.glob["redis.port"]), db=conf.glob["redis.db"]
)
rip = Ripper(redis_conn, conf.glob["outdir"])
# Main runner.
while True:
conf.read()
for subreddit in conf.subreddits():
rip.process(subreddit)
def execute(self, options):
log.info("Starting reddrip")
conf = Configuration(options.config)
redis_conn = redis.StrictRedis(
host=conf.glob["redis.host"],
port=int(conf.glob["redis.port"]),
db=conf.glob["redis.db"],
)
rip = Ripper(redis_conn, conf.glob["outdir"])
# Main runner.
while True:
conf.read()
for subreddit in conf.subreddits():
rip.process(subreddit)
def execute(self, options):
conf = Configuration(options.config)
redis_conn = redis.StrictRedis(
host=conf.glob["redis.host"],
port=int(conf.glob["redis.port"]),
db=conf.glob["redis.db"],
)
subreddits = redis_conn.smembers("subreddits")
print "Total of %d subreddits in database, following %s." % (
len(subreddits), len(conf))
print
table = Texttable(max_width=0)
table.set_deco(Texttable.HEADER)
table.header(
["Name", "Type", "Processed", "Saved", "Existing", "Size"])
table.set_cols_align(["l", "l", "r", "r", "r", "r"])
size = processed = saved = existing = 0
for subreddit in subreddits:
try:
dl_type = conf.subreddit(subreddit)["type"]
except KeyError:
continue
sub_processed = len(
redis_conn.smembers("stat:%s:processed:all" % subreddit))
sub_saved = self._get_from_redis(redis_conn,
"stat:%s:saved:count" % subreddit)
sub_size = self._get_from_redis(redis_conn,
"stat:%s:saved:size" % subreddit)
subdir = "%s/%s" % (conf.glob["outdir"], subreddit)
if os.path.exists(subdir):
sub_existing = len([
f for f in os.listdir(subdir)
if os.path.isfile("%s/%s" % (subdir, f))
])
else:
sub_existing = 0
processed += int(sub_processed)
saved += int(sub_saved)
size += int(sub_size)
existing += sub_existing
table.add_row([
subreddit, dl_type, sub_processed, sub_saved, sub_existing,
self._human_readable(sub_size)
])
table.add_row([
"TOTAL", "", processed, saved, existing,
self._human_readable(size)
])
print table.draw()
def execute(self, options):
conf = Configuration(options.config)
redis_conn = redis.StrictRedis(
host=conf.glob["redis.host"],
port=int(conf.glob["redis.port"]),
db=conf.glob["redis.db"],
)
subreddits = redis_conn.smembers("subreddits")
print "Total of %d subreddits in database, following %s." % (
len(subreddits), len(conf)
)
print
table = Texttable(max_width=0)
table.set_deco(Texttable.HEADER)
table.header([
"Name",
"Type",
"Processed",
"Saved",
"Existing",
"Size"
])
table.set_cols_align(["l", "l", "r", "r", "r", "r"])
size = processed = saved = existing = 0
for subreddit in subreddits:
try:
dl_type = conf.subreddit(subreddit)["type"]
except KeyError:
continue
sub_processed = len(
redis_conn.smembers("stat:%s:processed:all" % subreddit)
)
sub_saved = self._get_from_redis(
redis_conn, "stat:%s:saved:count" % subreddit
)
sub_size = self._get_from_redis(
redis_conn, "stat:%s:saved:size" % subreddit
)
subdir = "%s/%s" % (conf.glob["outdir"], subreddit)
if os.path.exists(subdir):
sub_existing = len([
f for f in os.listdir(subdir)
if os.path.isfile("%s/%s" % (subdir, f))
])
else:
sub_existing = 0
processed += int(sub_processed)
saved += int(sub_saved)
size += int(sub_size)
existing += sub_existing
table.add_row([
subreddit,
dl_type,
sub_processed,
sub_saved,
sub_existing,
self._human_readable(sub_size)
])
table.add_row([
"TOTAL",
"",
processed,
saved,
existing,
self._human_readable(size)
])
print table.draw()