def handle(self, *args, **options):
command = options['command']
actions = ('force', 'list', 'remove')
if not any(options[i] for i in actions):
msg = 'Please, provide one action: %s' % '|'.join(actions)
raise CommandError(msg)
# `cover` command do not use the `days` nor `hours` parameter
if options['days'] is None and options['hours'] is None \
and command not in ('cover',):
raise CommandError('Provide some days/hours to find old objects.')
elif options['days'] or options['hours']:
hours = 24 * int(options['days'] if options['days'] else 0)
hours += int(options['hours'] if options['hours'] else 0)
sources = self._get_sources(options['spiders'])
remove = options['remove']
list_ = options['list']
force = options['force']
list_ = list_ or not force
loglevel = options['loglevel']
logger.setLevel(loglevel)
if command == 'manga':
self._clean_manga(hours, sources, list_)
elif command == 'user':
self._clean_user(hours, remove, list_)
elif command == 'image-cache':
cache = os.path.join(settings.IMAGES_STORE, 'full')
self._clean_image_cache(hours, cache, list_)
elif command == 'mobi-cache':
cache = MobiCache(settings.MOBI_STORE)
self._clean_cache(hours, cache, list_)
elif command == 'issue-cache':
cache = IssueCache(settings.ISSUES_STORE, settings.IMAGES_STORE)
self._clean_cache(hours, cache, list_)
mobi_cache = MobiCache(settings.MOBI_STORE)
self._clean_broken_issue_cache(cache, mobi_cache, list_)
elif command == 'cover':
self._clean_cover(sources, list_)
elif command == 'result-processing':
self._clean_result(hours, Result.PROCESSING, list_)
elif command == 'result-failed':
self._clean_result(hours, Result.FAILED, list_)
else:
raise CommandError('Not valid command value.')
def test_cache(self):
self.cache['url1'] = ['tests/fixtures/cache/mobi1.mobi']
self.cache['url2'] = [
'tests/fixtures/cache/mobi2.1.mobi',
'tests/fixtures/cache/mobi2.2.mobi'
]
self.cache['url3'] = ['tests/fixtures/cache/mobi3.mobi']
self.assertTrue(len(self.cache) == 3)
md5 = hashlib.md5('url1').hexdigest()
self.assertEqual(
self.cache['url1'][0],
[('mobi1.mobi', 'tests/fixtures/tmp/data/%s-00' % md5)])
for key in self.cache:
self.assertTrue(key in ('url1', 'url2', 'url3'))
v = self.cache[key]
if key in ('url1', 'url3'):
self.assertTrue(len(v[0]) == 1)
else:
self.assertTrue(len(v[0]) == 2)
del self.cache['url1']
self.assertTrue(len(self.cache) == 2)
self.assertTrue('url1' not in self.cache)
self.assertTrue('url2' in self.cache)
self.assertTrue('url3' in self.cache)
# Test that the store directory is not wipeout
cache = MobiCache(self.cache.store)
self.assertTrue(len(self.cache) == 2)
self.assertTrue(len(cache) == 2)
def create_mobi(self):
"""Create the MOBI file and return a list of files and names."""
cache = MobiCache(settings.MOBI_STORE)
if self.issue.url not in cache:
mobi_and_containers = self._create_mobi()
# XXX TODO - We are not storing stats in the cache anymore (is
# not the place), so we need to store it in a different place.
# Maybe in the database?
cache[self.issue.url] = [m[0] for m in mobi_and_containers]
# The containers need to be cleaned here.
for _, container in mobi_and_containers:
container.clean()
mobi_info, _ = cache[self.issue.url]
return mobi_info
def send_mobi(issue, user):
"""RQ job to send MOBI documents."""
mobi_cache = MobiCache(settings.MOBI_STORE)
try:
result = issue.create_result_if_needed(user, Result.PROCESSING)
except Subscription.DoesNotExist:
# Results in PROCESSING status are cleaned during the
# subscription removal
msg = 'Subscription removed for user %s (%s)' % (user, issue)
logger.warning(msg)
return
if issue.url not in mobi_cache:
logger.error('Issue not found in mobi cache (%s)' % issue)
result.set_status(Result.FAILED)
return
# Ignore the creation date from the cache.
mobi_info, _ = mobi_cache[issue.url]
email = user.userprofile.email_kindle
for mobi_name, mobi_file in mobi_info:
try:
EmailMessage(subject='Your kmanga.net request',
body='',
from_email=settings.KMANGA_EMAIL,
to=[email],
attachments=[(mobi_name, open(mobi_file, 'rb').read(),
'application/x-mobipocket-ebook')
]).send()
except Exception:
logger.error('Error while sending issue (%s) to (%s)' %
(issue, user))
result.set_status(Result.FAILED)
else:
result.set_status(Result.SENT)
def setUp(self):
self.cache = MobiCache('tests/fixtures/tmp')
class TestMobiCache(unittest.TestCase):
def setUp(self):
self.cache = MobiCache('tests/fixtures/tmp')
def tearDown(self):
shutil.rmtree('tests/fixtures/tmp')
def test_cache(self):
self.cache['url1'] = ['tests/fixtures/cache/mobi1.mobi']
self.cache['url2'] = [
'tests/fixtures/cache/mobi2.1.mobi',
'tests/fixtures/cache/mobi2.2.mobi'
]
self.cache['url3'] = ['tests/fixtures/cache/mobi3.mobi']
self.assertTrue(len(self.cache) == 3)
md5 = hashlib.md5('url1').hexdigest()
self.assertEqual(
self.cache['url1'][0],
[('mobi1.mobi', 'tests/fixtures/tmp/data/%s-00' % md5)])
for key in self.cache:
self.assertTrue(key in ('url1', 'url2', 'url3'))
v = self.cache[key]
if key in ('url1', 'url3'):
self.assertTrue(len(v[0]) == 1)
else:
self.assertTrue(len(v[0]) == 2)
del self.cache['url1']
self.assertTrue(len(self.cache) == 2)
self.assertTrue('url1' not in self.cache)
self.assertTrue('url2' in self.cache)
self.assertTrue('url3' in self.cache)
# Test that the store directory is not wipeout
cache = MobiCache(self.cache.store)
self.assertTrue(len(self.cache) == 2)
self.assertTrue(len(cache) == 2)
def test_clean(self):
self.cache['url1'] = ['tests/fixtures/cache/mobi1.mobi']
self.cache['url2'] = ['tests/fixtures/cache/mobi2.1.mobi']
time.sleep(2)
self.assertTrue(len(self.cache) == 2)
self.cache['url3'] = ['tests/fixtures/cache/mobi3.mobi']
self.assertTrue(len(self.cache) == 3)
self.cache.clean(ttl=1)
self.assertTrue(len(self.cache) == 1)
self.assertTrue('url1' not in self.cache)
self.assertTrue('url2' not in self.cache)
self.assertTrue('url3' in self.cache)
del self.cache['url3']
self.assertTrue(len(self.cache) == 0)
self.cache['url1'] = ['tests/fixtures/cache/mobi1.mobi']
self.cache['url2'] = ['tests/fixtures/cache/mobi2.1.mobi']
time.sleep(2)
self.assertTrue(len(self.cache) == 2)
self.cache['url2'] = ['tests/fixtures/cache/mobi2.2.mobi']
self.cache['url3'] = ['tests/fixtures/cache/mobi3.mobi']
self.assertTrue(len(self.cache) == 3)
self.cache.clean(ttl=1)
self.assertTrue(len(self.cache) == 2)
self.assertTrue('url1' not in self.cache)
self.assertTrue('url2' in self.cache)
self.assertTrue('url3' in self.cache)
def test_free(self):
self.cache.slots = 3
self.cache.nclean = 2
self.cache['url1'] = ['tests/fixtures/cache/mobi1.mobi']
self.cache['url2.1'] = ['tests/fixtures/cache/mobi2.1.mobi']
self.cache.free()
self.assertTrue(len(self.cache) == 2)
self.cache['url2.2'] = ['tests/fixtures/cache/mobi2.2.mobi']
self.cache['url3'] = ['tests/fixtures/cache/mobi3.mobi']
self.cache.free()
self.assertTrue(len(self.cache) == 2)
self.assertTrue('url1' not in self.cache)
self.assertTrue('url2.1' not in self.cache)
self.assertTrue('url2.2' in self.cache)
self.assertTrue('url3' in self.cache)