def reviewer_sign_file(self):
"""Sign the original file (`file_path`) with reviewer certs, then move
the signed file to the reviewers-specific signed path
(`reviewer_signed_file_path`) on private storage."""
if not self.extension.uuid:
raise SigningError('Need uuid to be set to sign')
if not self.pk:
raise SigningError('Need version pk to be set to sign')
ids = json.dumps({
'id': self.review_id,
'version': self.pk
})
with statsd.timer('extensions.sign_reviewer'):
try:
# This will read the file from self.file_path, generate a
# reviewer signature and write the signed file to
# self.reviewer_signed_file_path.
sign_app(private_storage.open(self.file_path),
self.reviewer_signed_file_path, ids, reviewer=True)
except SigningError:
log.info(
'[ExtensionVersion:%s] Reviewer Signing failed' % self.pk)
if private_storage.exists(self.reviewer_signed_file_path):
private_storage.delete(self.reviewer_signed_file_path)
raise
def rmtree(prefix):
dirs, files = private_storage.listdir(prefix)
for fname in files:
private_storage.delete(os.path.join(prefix, fname))
for d in dirs:
rmtree(os.path.join(prefix, d))
private_storage.delete(prefix)
def delete(self):
log.info(u'Version deleted: %r (%s)' % (self, self.id))
mkt.log(mkt.LOG.DELETE_VERSION, self.addon, str(self.version))
models.signals.pre_delete.send(sender=Version, instance=self)
was_current = False
if self == self.addon.current_version:
was_current = True
self.update(deleted=True)
# Set file status to disabled.
f = self.all_files[0]
f.update(status=mkt.STATUS_DISABLED, _signal=False)
f.hide_disabled_file()
# If version deleted was the current version and there now exists
# another current_version, we need to call some extra methods to update
# various bits for packaged apps.
if was_current and self.addon.current_version:
self.addon.update_name_from_package_manifest()
self.addon.update_supported_locales()
if self.addon.is_packaged:
# Unlink signed packages if packaged app.
public_storage.delete(f.signed_file_path)
log.info(u'Unlinked file: %s' % f.signed_file_path)
private_storage.delete(f.signed_reviewer_file_path)
log.info(u'Unlinked file: %s' % f.signed_reviewer_file_path)
models.signals.post_delete.send(sender=Version, instance=self)
def test_delete_mid_read(self):
self.viewer.extract()
self.viewer.select('install.js')
private_storage.delete(os.path.join(self.viewer.dest, 'install.js'))
res = self.viewer.read_file()
eq_(res, '')
assert self.viewer.selected['msg'].startswith('That file no')
def dump_user_installs_cron():
"""
Sets up tasks to do user install dumps.
"""
chunk_size = 100
# Get valid users to dump.
user_ids = set(Installed.objects.filter(user__enable_recommendations=True)
.values_list('user', flat=True))
# Clean up the path where we'll store the individual json files from each
# user installs dump (which are in users/ in DUMPED_USERS_PATH).
path_to_cleanup = os.path.join(settings.DUMPED_USERS_PATH, 'users')
task_log.info('Cleaning up path {0}'.format(path_to_cleanup))
try:
for dirpath, dirnames, filenames in walk_storage(
path_to_cleanup, storage=private_storage):
for filename in filenames:
private_storage.delete(os.path.join(dirpath, filename))
except OSError:
# Ignore if the directory does not exist.
pass
grouping = []
for chunk in chunked(user_ids, chunk_size):
grouping.append(dump_user_installs.subtask(args=[chunk]))
post = zip_users.subtask(immutable=True)
ts = chord(grouping, post)
ts.apply_async()
def resize_preview(src, pk, **kw):
"""Resizes preview images and stores the sizes on the preview."""
instance = Preview.objects.get(pk=pk)
thumb_dst, full_dst = instance.thumbnail_path, instance.image_path
sizes = instance.sizes or {}
log.info("[1@None] Resizing preview and storing size: %s" % thumb_dst)
try:
thumbnail_size = APP_PREVIEW_SIZES[0][:2]
image_size = APP_PREVIEW_SIZES[1][:2]
with private_storage.open(src, "rb") as fp:
size = Image.open(fp).size
if size[0] > size[1]:
# If the image is wider than tall, then reverse the wanted size
# to keep the original aspect ratio while still resizing to
# the correct dimensions.
thumbnail_size = thumbnail_size[::-1]
image_size = image_size[::-1]
if kw.get("generate_thumbnail", True):
sizes["thumbnail"] = resize_image(src, thumb_dst, thumbnail_size, remove_src=False)
if kw.get("generate_image", True):
sizes["image"] = resize_image(src, full_dst, image_size, remove_src=False)
instance.sizes = sizes
instance.save()
log.info("Preview resized to: %s" % thumb_dst)
# Remove src file now that it has been processed.
private_storage.delete(src)
return True
except Exception, e:
log.error("Error saving preview: %s; %s" % (e, thumb_dst))
def reviewer_sign_file(self):
"""Sign the original file (`file_path`) with reviewer certs, then move
the signed file to the reviewers-specific signed path
(`reviewer_signed_file_path`) on private storage."""
if not self.extension.uuid:
raise SigningError('Need uuid to be set to sign')
if not self.pk:
raise SigningError('Need version pk to be set to sign')
ids = json.dumps({
# Reviewers get a unique 'id' so the reviewer installed add-on
# won't conflict with the public add-on, and also so even multiple
# versions of the same add-on can be installed side by side with
# other versions.
'id': 'reviewer-{guid}-{version_id}'.format(
guid=self.extension.uuid, version_id=self.pk),
'version': self.pk
})
with statsd.timer('extensions.sign_reviewer'):
try:
# This will read the file from self.file_path, generate a
# reviewer signature and write the signed file to
# self.reviewer_signed_file_path.
sign_app(private_storage.open(self.file_path),
self.reviewer_signed_file_path, ids, reviewer=True)
except SigningError:
log.info(
'[ExtensionVersion:%s] Reviewer Signing failed' % self.pk)
if private_storage.exists(self.reviewer_signed_file_path):
private_storage.delete(self.reviewer_signed_file_path)
raise
def delete(self):
log.info(u'Version deleted: %r (%s)' % (self, self.id))
mkt.log(mkt.LOG.DELETE_VERSION, self.webapp, str(self.version))
models.signals.pre_delete.send(sender=Version, instance=self)
was_current = False
if self == self.webapp.current_version:
was_current = True
self.update(deleted=True)
# Set file status to disabled.
f = self.all_files[0]
f.update(status=mkt.STATUS_DISABLED, _signal=False)
f.hide_disabled_file()
# If version deleted was the current version and there now exists
# another current_version, we need to call some extra methods to update
# various bits for packaged apps.
if was_current and self.webapp.current_version:
self.webapp.update_name_from_package_manifest()
self.webapp.update_supported_locales()
if self.webapp.is_packaged:
# Unlink signed packages if packaged app.
public_storage.delete(f.signed_file_path)
log.info(u'Unlinked file: %s' % f.signed_file_path)
private_storage.delete(f.signed_reviewer_file_path)
log.info(u'Unlinked file: %s' % f.signed_reviewer_file_path)
models.signals.post_delete.send(sender=Version, instance=self)
def rmtree(prefix):
dirs, files = private_storage.listdir(prefix)
for fname in files:
private_storage.delete(os.path.join(prefix, fname))
for d in dirs:
rmtree(os.path.join(prefix, d))
private_storage.delete(prefix)
def cleanup_extracted_file():
log.info('Removing extracted files for file viewer.')
root = os.path.join(settings.TMP_PATH, 'file_viewer')
# Local storage uses local time for file modification. S3 uses UTC time.
now = datetime.utcnow if storage_is_remote() else datetime.now
for path in private_storage.listdir(root)[0]:
full = os.path.join(root, path)
age = now() - private_storage.modified_time(
os.path.join(full, 'manifest.webapp'))
if age.total_seconds() > (60 * 60):
log.debug('Removing extracted files: %s, %dsecs old.' %
(full, age.total_seconds()))
for subroot, dirs, files in walk_storage(full):
for f in files:
private_storage.delete(os.path.join(subroot, f))
# Nuke out the file and diff caches when the file gets removed.
id = os.path.basename(path)
try:
int(id)
except ValueError:
continue
key = hashlib.md5()
key.update(str(id))
cache.delete('%s:memoize:%s:%s' % (settings.CACHE_PREFIX,
'file-viewer', key.hexdigest()))
def test_bom(self):
dest = os.path.join(settings.TMP_PATH, 'test_bom')
with private_storage.open(dest, 'w') as f:
f.write('foo'.encode('utf-16'))
self.viewer.select('foo')
self.viewer.selected = {'full': dest, 'size': 1}
eq_(self.viewer.read_file(), u'foo')
private_storage.delete(dest)
def test_view_one_missing(self):
self.file_viewer.extract()
private_storage.delete(
os.path.join(self.file_viewer.right.dest, 'script.js'))
res = self.client.get(self.file_url(not_binary))
doc = pq(res.content)
eq_(len(doc('pre')), 3)
eq_(len(doc('#content-wrapper p')), 2)
def test_different_tree(self):
self.file_viewer.extract()
private_storage.delete(
os.path.join(self.file_viewer.left.dest, not_binary))
res = self.client.get(self.file_url(not_binary))
doc = pq(res.content)
eq_(doc('h4:last').text(), 'Deleted files:')
eq_(len(doc('ul.root')), 2)
def test_view_one_missing(self):
self.file_viewer.extract()
private_storage.delete(os.path.join(self.file_viewer.right.dest,
'script.js'))
res = self.client.get(self.file_url(not_binary))
doc = pq(res.content)
eq_(len(doc('pre')), 3)
eq_(len(doc('#content-wrapper p')), 2)
def test_different_tree(self):
self.file_viewer.extract()
private_storage.delete(os.path.join(self.file_viewer.left.dest,
not_binary))
res = self.client.get(self.file_url(not_binary))
doc = pq(res.content)
eq_(doc('h4:last').text(), 'Deleted files:')
eq_(len(doc('ul.root')), 2)
def setup_files(self):
# Clean out any left over stuff.
private_storage.delete(self.file.signed_file_path)
private_storage.delete(self.file.signed_reviewer_file_path)
# Make sure the source file is there.
if not private_storage.exists(self.file.file_path):
copy_to_storage(self.packaged_app_path('mozball.zip'),
self.file.file_path)
def cleanup(self):
try:
for root, dirs, files in walk_storage(
self.dest, storage=private_storage):
for fname in files:
private_storage.delete(os.path.join(root, fname))
except OSError as e:
if e.errno == 2:
# Directory doesn't exist, nothing to clean up.
return
raise
def cleanup(self):
try:
for root, dirs, files in walk_storage(self.dest,
storage=private_storage):
for fname in files:
private_storage.delete(os.path.join(root, fname))
except OSError as e:
if e.errno == 2:
# Directory doesn't exist, nothing to clean up.
return
raise
def generate_previews(app, n=1):
gen = pydenticon.Generator(8, 12, foreground=foreground, digest=hashlib.sha512)
for i in range(n):
img = gen.generate(unicode(app.name) + unichr(i), 320, 480, output_format="png")
p = Preview.objects.create(addon=app, filetype="image/png", caption="screenshot " + str(i), position=i)
fn = tempfile.mktemp()
try:
f = private_storage.open(fn, "w")
f.write(img)
f.close()
resize_preview(fn, p.pk)
finally:
private_storage.delete(fn)
def clean_old_signed(seconds=60 * 60):
"""Clean out apps signed for reviewers."""
log.info('Removing old apps signed for reviewers')
root = settings.SIGNED_APPS_REVIEWER_PATH
# Local storage uses local time for file modification. S3 uses UTC time.
now = datetime.utcnow if storage_is_remote() else datetime.now
for nextroot, dirs, files in walk_storage(root):
for fn in files:
full = os.path.join(nextroot, fn)
age = now() - private_storage.modified_time(full)
if age.total_seconds() > seconds:
log.debug('Removing signed app: %s, %dsecs old.' % (
full, age.total_seconds()))
private_storage.delete(full)
def test_delete_with_file(self):
"""Test that when a Extension instance is deleted, the corresponding
file on the filesystem is also deleted."""
extension = Extension.objects.create(version='0.1')
file_path = extension.file_path
with private_storage.open(file_path, 'w') as f:
f.write('sample data\n')
assert private_storage.exists(file_path)
try:
extension.delete()
assert not private_storage.exists(file_path)
finally:
if private_storage.exists(file_path):
private_storage.delete(file_path)
def mkt_gc(**kw):
"""Site-wide garbage collections."""
log.info('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=mkt.LOG_KEEP).values_list('id', flat=True))
for chunk in chunked(logs, 100):
chunk.sort()
log.info('Deleting log entries: %s' % str(chunk))
delete_logs.delay(chunk)
# Clear oauth nonce rows. These expire after 10 minutes but we're just
# clearing those that are more than 1 day old.
Nonce.objects.filter(created__lt=days_ago(1)).delete()
# Delete the dump apps over 30 days.
_remove_stale_files(os.path.join(settings.DUMPED_APPS_PATH, 'tarballs'),
settings.DUMPED_APPS_DAYS_DELETE,
'Deleting old tarball: {0}',
storage=public_storage)
# Delete the dumped user installs over 30 days. Those are using private
# storage.
_remove_stale_files(os.path.join(settings.DUMPED_USERS_PATH, 'tarballs'),
settings.DUMPED_USERS_DAYS_DELETE,
'Deleting old tarball: {0}',
storage=private_storage)
# Delete old files in select directories under TMP_PATH.
_remove_stale_files(os.path.join(settings.TMP_PATH, 'preview'),
settings.TMP_PATH_DAYS_DELETE,
'Deleting TMP_PATH file: {0}',
storage=private_storage)
_remove_stale_files(os.path.join(settings.TMP_PATH, 'icon'),
settings.TMP_PATH_DAYS_DELETE,
'Deleting TMP_PATH file: {0}',
storage=private_storage)
# Delete stale FileUploads.
for fu in FileUpload.objects.filter(created__lte=days_ago(90)):
log.debug(u'[FileUpload:{uuid}] Removing file: {path}'
.format(uuid=fu.uuid, path=fu.path))
if fu.path:
try:
private_storage.delete(fu.path)
except OSError:
pass
fu.delete()
def export_data(name=None):
today = datetime.datetime.today().strftime('%Y-%m-%d')
if name is None:
name = today
# Clean up the path where we'll store the individual json files from each
# app dump.
for dirpath, dirnames, filenames in walk_storage(
settings.DUMPED_APPS_PATH, storage=private_storage):
for filename in filenames:
private_storage.delete(os.path.join(dirpath, filename))
task_log.info('Cleaning up path {0}'.format(settings.DUMPED_APPS_PATH))
# Run all dump_apps task in parallel, and once it's done, add extra files
# and run compression.
chord(dump_all_apps_tasks(),
compress_export.si(tarball_name=name, date=today)).apply_async()
def cleanup_file(sender, instance, **kw):
""" On delete of the file object from the database, unlink the file from
the file system """
if kw.get('raw') or not instance.filename:
return
# Use getattr so the paths are accessed inside the try block.
for path in ('file_path', 'guarded_file_path', 'signed_file_path'):
try:
filename = getattr(instance, path, None)
except models.ObjectDoesNotExist:
return
if filename and (public_storage.exists(filename) or
private_storage.exists(filename)):
log.info('Removing filename: %s for file: %s'
% (filename, instance.pk))
public_storage.delete(filename)
private_storage.delete(filename)
def generate_previews(app, n=1):
gen = pydenticon.Generator(8, 12, foreground=foreground,
digest=hashlib.sha512)
for i in range(n):
img = gen.generate(unicode(app.name) + unichr(i), 320, 480,
output_format="png")
p = Preview.objects.create(addon=app, filetype="image/png",
caption="screenshot " + str(i),
position=i)
fn = tempfile.mktemp()
try:
f = private_storage.open(fn, 'w')
f.write(img)
f.close()
resize_preview(fn, p.pk)
finally:
private_storage.delete(fn)
def export_data(name=None):
today = datetime.datetime.today().strftime('%Y-%m-%d')
if name is None:
name = today
# Clean up the path where we'll store the individual json files from each
# app dump.
for dirpath, dirnames, filenames in walk_storage(settings.DUMPED_APPS_PATH,
storage=private_storage):
for filename in filenames:
private_storage.delete(os.path.join(dirpath, filename))
task_log.info('Cleaning up path {0}'.format(settings.DUMPED_APPS_PATH))
# Run all dump_apps task in parallel, and once it's done, add extra files
# and run compression.
chord(dump_all_apps_tasks(),
compress_export.si(tarball_name=name, date=today)).apply_async()
def resize_promo_imgs(src, dst, sizes, **kw):
"""Resizes webapp/website promo imgs."""
log.info('[1@None] Resizing promo imgs: %s' % dst)
try:
for s in sizes:
size_dst = '%s-%s.png' % (dst, s)
# Crop only to the width, keeping the aspect ratio.
resize_image(src, size_dst, (s, 0), remove_src=False)
pngcrush_image.delay(size_dst, **kw)
with private_storage.open(src) as fd:
promo_img_hash = _hash_file(fd)
private_storage.delete(src)
log.info('Promo img hash resizing completed for: %s' % dst)
return {'promo_img_hash': promo_img_hash}
except Exception, e:
log.error("Error resizing promo img hash: %s; %s" % (e, dst))
def test_delete_with_file(self):
"""Test that when a Extension instance is deleted, the ExtensionVersion
referencing it are also deleted, as well as the attached files."""
extension = Extension.objects.create()
version = ExtensionVersion.objects.create(
extension=extension, version='0.1')
file_path = version.file_path
with private_storage.open(file_path, 'w') as f:
f.write('sample data\n')
assert private_storage.exists(file_path)
try:
extension.delete()
assert not Extension.objects.count()
assert not ExtensionVersion.objects.count()
assert not private_storage.exists(file_path)
finally:
if private_storage.exists(file_path):
private_storage.delete(file_path)
def cleanup_file(sender, instance, **kw):
""" On delete of the file object from the database, unlink the file from
the file system """
if kw.get('raw') or not instance.filename:
return
# Use getattr so the paths are accessed inside the try block.
for path in ('file_path', 'guarded_file_path', 'reviewer_signed_file_path',
'signed_file_path'):
try:
filename = getattr(instance, path, None)
except models.ObjectDoesNotExist:
return
if filename and (public_storage.exists(filename) or
private_storage.exists(filename)):
log.info('Removing filename: %s for file: %s'
% (filename, instance.pk))
public_storage.delete(filename)
private_storage.delete(filename)
def from_upload(cls, upload, webapp, send_signal=True):
data = utils.parse_webapp(upload, webapp)
max_len = cls._meta.get_field_by_name('_developer_name')[0].max_length
developer = data.get('developer_name', '')[:max_len]
v = cls.objects.create(webapp=webapp, version=data['version'],
_developer_name=developer)
log.info('New version: %r (%s) from %r' % (v, v.id, upload))
# To avoid circular import.
from mkt.webapps.models import AppManifest
# Note: This must happen before we call `File.from_upload`.
manifest = utils.WebAppParser().get_json_data(upload)
AppManifest.objects.create(
version=v, manifest=json.dumps(manifest))
File.from_upload(upload, v, parse_data=data)
# Update supported locales from manifest.
# Note: This needs to happen after we call `File.from_upload`.
update_supported_locales_single.apply_async(
args=[webapp.id], kwargs={'latest': True},
eta=datetime.datetime.now() +
datetime.timedelta(seconds=settings.NFS_LAG_DELAY)
)
v.disable_old_files()
# After the upload has been copied, remove the upload.
private_storage.delete(upload.path)
if send_signal:
version_uploaded.send(sender=v)
# If packaged app and app is blocked, put in escalation queue.
if webapp.is_packaged and webapp.status == mkt.STATUS_BLOCKED:
# To avoid circular import.
from mkt.reviewers.models import EscalationQueue
EscalationQueue.objects.create(webapp=webapp)
return v
def export_data(name=None):
today = datetime.datetime.today().strftime('%Y-%m-%d')
if name is None:
name = today
# Clean up the path where we'll store the individual json files from each
# app dump (which are in apps/ inside DUMPED_APPS_PATH).
path_to_cleanup = os.path.join(settings.DUMPED_APPS_PATH, 'apps')
task_log.info('Cleaning up path {0}'.format(settings.DUMPED_APPS_PATH))
try:
for dirpath, dirnames, filenames in walk_storage(
path_to_cleanup, storage=private_storage):
for filename in filenames:
private_storage.delete(os.path.join(dirpath, filename))
except OSError:
# Ignore if the directory does not exist.
pass
# Run all dump_apps task in parallel, and once it's done, add extra files
# and run compression.
chord(dump_all_apps_tasks(),
compress_export.si(tarball_name=name, date=today)).apply_async()
def from_upload(cls, upload, addon, send_signal=True):
data = utils.parse_addon(upload, addon)
max_len = cls._meta.get_field_by_name('_developer_name')[0].max_length
developer = data.get('developer_name', '')[:max_len]
v = cls.objects.create(addon=addon, version=data['version'],
_developer_name=developer)
log.info('New version: %r (%s) from %r' % (v, v.id, upload))
# To avoid circular import.
from mkt.webapps.models import AppManifest
# Note: This must happen before we call `File.from_upload`.
manifest = utils.WebAppParser().get_json_data(upload)
AppManifest.objects.create(
version=v, manifest=json.dumps(manifest))
File.from_upload(upload, v, parse_data=data)
# Update supported locales from manifest.
# Note: This needs to happen after we call `File.from_upload`.
update_supported_locales_single.apply_async(
args=[addon.id], kwargs={'latest': True},
eta=datetime.datetime.now() +
datetime.timedelta(seconds=settings.NFS_LAG_DELAY)
)
v.disable_old_files()
# After the upload has been copied, remove the upload.
private_storage.delete(upload.path)
if send_signal:
version_uploaded.send(sender=v)
# If packaged app and app is blocked, put in escalation queue.
if addon.is_packaged and addon.status == mkt.STATUS_BLOCKED:
# To avoid circular import.
from mkt.reviewers.models import EscalationQueue
EscalationQueue.objects.create(addon=addon)
return v
def tearDown(self):
self.helper.cleanup()
if storage_is_remote():
private_storage.delete(self.packaged_app_path('signed.zip'))
def tearDown(self):
private_storage.delete(self.tmp_good)
private_storage.delete(self.tmp_bad)
super(TestTask, self).tearDown()
def test_diffable_deleted_files(self):
self.helper.extract()
private_storage.delete(os.path.join(self.helper.left.dest,
'index.html'))
eq_('index.html' in self.helper.get_deleted_files(), True)
def tearDown(self):
private_storage.delete(self.tmp_good)
private_storage.delete(self.tmp_bad)
super(TestTask, self).tearDown()
def test_diffable_one_missing(self):
self.helper.extract()
private_storage.delete(os.path.join(self.helper.right.dest,
'index.html'))
self.helper.select('index.html')
assert self.helper.is_diffable()
def tearDown(self):
for tmp in self.tmp_files:
private_storage.delete(tmp)
