def setUp(self):
fn = get_file('dictionary-test.xpi')
if storage_is_remote():
copy_stored_file(
fn, fn,
src_storage=local_storage, dst_storage=private_storage)
self.viewer = FileViewer(make_file(1, fn))
def setUp(self):
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True, status=mkt.WEBAPPS_UNREVIEWED_STATUS)
self.dev = self.app.authors.all()[0]
self.regular = UserProfile.objects.get(pk=999)
self.version = self.app.versions.latest()
self.file = self.version.all_files[0]
self.versions = [self.version,
self.app.versions.create(
version='%s.1' % self.version.version)]
self.files = [self.file,
File.objects.create(version=self.versions[1],
filename='webapp.zip')]
self.login_as_editor()
for file_obj in self.files:
src = os.path.join(settings.ROOT, packaged_app)
if file_obj.status in mkt.LISTED_STATUSES:
target = public_storage
else:
target = private_storage
copy_stored_file(src, file_obj.file_path,
src_storage=local_storage,
dst_storage=target)
self.file_viewer = FileViewer(self.file)
def from_upload(cls, upload, version, parse_data={}):
upload.path = smart_path(nfd_str(upload.path))
ext = os.path.splitext(upload.path)[1]
f = cls(version=version)
f.filename = f.generate_filename(extension=ext or '.zip')
f.size = private_storage.size(upload.path) # Size in bytes.
f.status = mkt.STATUS_PENDING
# Re-use the file-upload hash if we can, no need to regenerate a new
# one if we can avoid that.
f.hash = upload.hash or f.generate_hash(upload.path)
f.save()
log.debug('New file: %r from %r' % (f, upload))
# Move the uploaded file from the temp location.
copy_stored_file(
upload.path,
os.path.join(version.path_prefix, nfd_str(f.filename)),
src_storage=private_storage,
dst_storage=private_storage)
if upload.validation:
FileValidation.from_json(f, upload.validation)
return f
def test_self_copy(self):
src = self.newfile('src.txt', '<contents>')
dst = self.path('src.txt')
copy_stored_file(
src, dst,
src_storage=private_storage, dst_storage=private_storage)
eq_(self.contents(dst), '<contents>')
def run_validator(file_path, url=None):
"""A pre-configured wrapper around the app validator."""
temp_path = None
# Make a copy of the file since we can't assume the
# uploaded file is on the local filesystem.
temp_path = tempfile.mktemp()
copy_stored_file(
file_path, temp_path,
src_storage=private_storage, dst_storage=local_storage)
with statsd.timer('mkt.developers.validator'):
is_packaged = zipfile.is_zipfile(temp_path)
if is_packaged:
log.info(u'Running `validate_packaged_app` for path: %s'
% (file_path))
with statsd.timer('mkt.developers.validate_packaged_app'):
return validate_packaged_app(
temp_path,
market_urls=settings.VALIDATOR_IAF_URLS,
timeout=settings.VALIDATOR_TIMEOUT,
spidermonkey=settings.SPIDERMONKEY)
else:
log.info(u'Running `validate_app` for path: %s' % (file_path))
with statsd.timer('mkt.developers.validate_app'):
return validate_app(open(temp_path).read(),
market_urls=settings.VALIDATOR_IAF_URLS,
url=url)
# Clean up copied files.
os.unlink(temp_path)
def run_validator(file_path, url=None):
"""A pre-configured wrapper around the app validator."""
temp_path = None
# Make a copy of the file since we can't assume the
# uploaded file is on the local filesystem.
temp_path = tempfile.mktemp()
copy_stored_file(
file_path, temp_path,
src_storage=private_storage, dst_storage=local_storage)
with statsd.timer('mkt.developers.validator'):
is_packaged = zipfile.is_zipfile(temp_path)
if is_packaged:
log.info(u'Running `validate_packaged_app` for path: %s'
% (file_path))
with statsd.timer('mkt.developers.validate_packaged_app'):
return validate_packaged_app(
temp_path,
market_urls=settings.VALIDATOR_IAF_URLS,
timeout=settings.VALIDATOR_TIMEOUT,
spidermonkey=settings.SPIDERMONKEY)
else:
log.info(u'Running `validate_app` for path: %s' % (file_path))
with statsd.timer('mkt.developers.validate_app'):
return validate_app(open(temp_path).read(),
market_urls=settings.VALIDATOR_IAF_URLS,
url=url)
# Clean up copied files.
os.unlink(temp_path)
def setUp(self):
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True, status=mkt.WEBAPPS_UNREVIEWED_STATUS)
self.dev = self.app.authors.all()[0]
self.regular = UserProfile.objects.get(pk=999)
self.version = self.app.versions.latest()
self.file = self.version.all_files[0]
self.versions = [
self.version,
self.app.versions.create(version='%s.1' % self.version.version)
]
self.files = [
self.file,
File.objects.create(version=self.versions[1],
filename='webapp.zip')
]
self.login_as_editor()
for file_obj in self.files:
src = os.path.join(settings.ROOT, packaged_app)
if file_obj.status in mkt.LISTED_STATUSES:
target = public_storage
else:
target = private_storage
copy_stored_file(src,
file_obj.file_path,
src_storage=local_storage,
dst_storage=target)
self.file_viewer = FileViewer(self.file)
def from_upload(cls, upload, version, parse_data={}):
upload.path = smart_path(nfd_str(upload.path))
ext = os.path.splitext(upload.path)[1]
f = cls(version=version)
f.filename = f.generate_filename(extension=ext or '.zip')
f.size = private_storage.size(upload.path) # Size in bytes.
f.status = mkt.STATUS_PENDING
# Re-use the file-upload hash if we can, no need to regenerate a new
# one if we can avoid that.
f.hash = upload.hash or f.generate_hash(upload.path)
f.save()
log.debug('New file: %r from %r' % (f, upload))
# Move the uploaded file from the temp location.
copy_stored_file(
upload.path,
os.path.join(version.path_prefix, nfd_str(f.filename)),
src_storage=private_storage,
dst_storage=private_storage)
if upload.validation:
FileValidation.from_json(f, upload.validation)
return f
def setUp(self):
src = self.packaged_app_path('signed.zip')
if storage_is_remote():
copy_stored_file(
src, src,
src_storage=local_storage, dst_storage=private_storage)
self.helper = DiffHelper(make_file(1, src), make_file(2, src))
def test_pngcrush_image_is_called(self, mock_move):
expected_suffix = '.opti.png'
tmp_src = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
tmp_dest = os.path.splitext(tmp_src.name)[0] + expected_suffix
copy_stored_file(self.img_path, tmp_dest, src_storage=public_storage,
dst_storage=local_storage)
with mock.patch('tempfile.NamedTemporaryFile',
lambda *a, **k: tmp_src):
rval = tasks.pngcrush_image(self.img_path)
# pngcrush_image copies the stored file to a local tempfile.
eq_(open(tmp_src.name).read(),
public_storage.open(self.img_path).read())
expected_cmd = ['pngcrush', '-q', '-rem', 'alla', '-brute', '-reduce',
'-e', expected_suffix, tmp_src.name]
# pngcrush_image incokes pngcrush with the tempfile name and writes to
# another tempfile.
self.mock_popen.assert_called_once_with(
expected_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
# The output file is then copied back to storage.
mock_move.assert_called_once_with(tmp_dest, self.img_path,
dst_storage=public_storage,
src_storage=local_storage)
eq_(rval, {'image_hash': 'bb362450'})
def setUp(self):
fn = get_file('dictionary-test.xpi')
if storage_is_remote():
copy_stored_file(fn,
fn,
src_storage=local_storage,
dst_storage=private_storage)
self.viewer = FileViewer(make_file(1, fn))
def test_self_copy(self):
src = self.newfile('src.txt', '<contents>')
dst = self.path('src.txt')
copy_stored_file(src,
dst,
src_storage=private_storage,
dst_storage=private_storage)
eq_(self.contents(dst), '<contents>')
def setUp(self):
src = self.packaged_app_path('signed.zip')
if storage_is_remote():
copy_stored_file(src,
src,
src_storage=local_storage,
dst_storage=private_storage)
self.helper = DiffHelper(make_file(1, src), make_file(2, src))
def test_non_ascii(self):
src = self.newfile(u'kristi\u0107.txt',
u'ivan kristi\u0107'.encode('utf8'))
dst = self.path(u'somedir/kristi\u0107.txt')
copy_stored_file(
src, dst,
src_storage=private_storage, dst_storage=private_storage)
eq_(self.contents(dst), 'ivan kristi\xc4\x87')
def test_copy_stored_file_when_local(self, mock):
tmp = tempfile.mkstemp()[1]
copy_stored_file(tmp,
tmp,
src_storage=local_storage,
dst_storage=private_storage)
assert not mock.called
local_storage.delete(tmp)
def test_non_ascii(self):
src = self.newfile(u'kristi\u0107.txt',
u'ivan kristi\u0107'.encode('utf8'))
dst = self.path(u'somedir/kristi\u0107.txt')
copy_stored_file(src,
dst,
src_storage=private_storage,
dst_storage=private_storage)
eq_(self.contents(dst), 'ivan kristi\xc4\x87')
def setUp(self):
self.webapp_path = tempfile.mktemp(suffix='.webapp')
copy_stored_file(
self.manifest_path('mozball.webapp'), self.webapp_path,
src_storage=local_storage, dst_storage=private_storage)
self.tmp_files = []
self.manifest = dict(name=u'Ivan Krsti\u0107', version=u'1.0',
description=u'summary',
developer=dict(name=u'Dev Namé'))
def test_no_manifest_at_root(self):
path = self.packaged_app_path('no-manifest-at-root.zip')
if storage_is_remote():
copy_stored_file(path, path, src_storage=local_storage,
dst_storage=private_storage)
with self.assertRaises(forms.ValidationError) as exc:
WebAppParser().parse(private_storage.open(path))
m = exc.exception.messages[0]
assert m.startswith('The file "manifest.webapp" was not found'), (
'Unexpected: %s' % m)
def test_disabled_but_owner(self):
self.login('*****@*****.**')
self.file.update(status=mkt.STATUS_DISABLED)
copy_stored_file(self.packaged_app_path('mozball.zip'),
self.file.file_path,
src_storage=local_storage,
dst_storage=private_storage)
path = private_storage.url(self.file.file_path)
res = self.client.get(self.url)
self.assert3xx(res, path)
def test_disabled_but_owner(self):
self.login('*****@*****.**')
self.file.update(status=mkt.STATUS_DISABLED)
copy_stored_file(self.packaged_app_path('mozball.zip'),
self.file.file_path,
src_storage=local_storage,
dst_storage=private_storage)
path = private_storage.url(self.file.file_path)
res = self.client.get(self.url)
self.assert3xx(res, path)
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name, 'position': 1})
copy_stored_file(
get_image_path(name), os.path.join(self.dest, name),
src_storage=local_storage, dst_storage=private_storage)
assert form.is_valid(), form.errors
form.save(self.addon)
eq_(self.addon.previews.all()[0].sizes,
{u'image': [250, 297], u'thumbnail': [100, 119]})
def setup_files(self):
# Clean out any left over stuff.
public_storage.delete(self.file.signed_file_path)
private_storage.delete(self.file.signed_reviewer_file_path)
# Make sure the source file is there.
if not private_storage.exists(self.file.file_path):
copy_stored_file(self.packaged_app_path('mozball.zip'),
self.file.file_path, src_storage=local_storage,
dst_storage=private_storage)
def compress_export(tarball_name, date):
# We need a temporary directory on the local filesystem that will contain
# all files in order to call `tar`.
local_source_dir = tempfile.mkdtemp()
apps_dirpath = os.path.join(settings.DUMPED_APPS_PATH, 'apps')
# In case apps_dirpath is empty, add a dummy file to make the apps
# directory in the tar archive non-empty. It should not happen in prod, but
# it's nice to have it to prevent the task from failing entirely.
with private_storage.open(os.path.join(apps_dirpath, '0', '.keep'),
'w') as fd:
fd.write('.')
# Now, copy content from private_storage to that temp directory. We don't
# need to worry about creating the directories locally, the storage class
# does that for us.
for dirpath, dirnames, filenames in walk_storage(apps_dirpath,
storage=private_storage):
for filename in filenames:
src_path = os.path.join(dirpath, filename)
dst_path = os.path.join(local_source_dir, 'apps',
os.path.basename(dirpath), filename)
copy_stored_file(src_path,
dst_path,
src_storage=private_storage,
dst_storage=local_storage)
# Also add extra files to the temp directory.
extra_filenames = compile_extra_files(local_source_dir, date)
# All our files are now present locally, let's generate a local filename
# that will contain the final '.tar.gz' before it's copied over to
# public storage.
local_target_file = tempfile.NamedTemporaryFile(suffix='.tgz',
prefix='dumped-apps-')
# tar ALL the things!
cmd = ['tar', 'czf', local_target_file.name, '-C', local_source_dir
] + ['apps'] + extra_filenames
task_log.info(u'Creating dump {0}'.format(local_target_file.name))
subprocess.call(cmd)
# Now copy the local tgz to the public storage.
remote_target_filename = os.path.join(settings.DUMPED_APPS_PATH,
'tarballs', '%s.tgz' % tarball_name)
copy_stored_file(local_target_file.name,
remote_target_filename,
src_storage=local_storage,
dst_storage=public_storage)
# Clean-up.
local_target_file.close()
rm_directory(local_source_dir)
return remote_target_filename
def setup_files(self):
# Clean out any left over stuff.
public_storage.delete(self.file.signed_file_path)
private_storage.delete(self.file.signed_reviewer_file_path)
# Make sure the source file is there.
if not private_storage.exists(self.file.file_path):
copy_stored_file(self.packaged_app_path('mozball.zip'),
self.file.file_path,
src_storage=local_storage,
dst_storage=private_storage)
def setUp(self):
self.webapp_path = tempfile.mktemp(suffix='.webapp')
copy_stored_file(self.manifest_path('mozball.webapp'),
self.webapp_path,
src_storage=local_storage,
dst_storage=private_storage)
self.tmp_files = []
self.manifest = dict(name=u'Ivan Krsti\u0107',
version=u'1.0',
description=u'summary',
developer=dict(name=u'Dev Namé'))
def test_no_manifest_at_root(self):
path = self.packaged_app_path('no-manifest-at-root.zip')
if storage_is_remote():
copy_stored_file(path,
path,
src_storage=local_storage,
dst_storage=private_storage)
with self.assertRaises(forms.ValidationError) as exc:
WebAppParser().parse(private_storage.open(path))
m = exc.exception.messages[0]
assert m.startswith('The file "manifest.webapp" was not found'), (
'Unexpected: %s' % m)
def get_image(self, filename):
"""Copy image to tmp and return tmp path.
We do this because the task `resize_preview` removes the src file when
finished.
"""
src = get_image_path(filename)
dst = os.path.join(settings.TMP_PATH, 'preview', filename)
copy_stored_file(
src, dst, src_storage=local_storage, dst_storage=private_storage)
return dst
def compress_export(tarball_name, date):
# We need a temporary directory on the local filesystem that will contain
# all files in order to call `tar`.
local_source_dir = tempfile.mkdtemp()
apps_dirpath = os.path.join(settings.DUMPED_APPS_PATH, 'apps')
# In case apps_dirpath is empty, add a dummy file to make the apps
# directory in the tar archive non-empty. It should not happen in prod, but
# it's nice to have it to prevent the task from failing entirely.
with private_storage.open(
os.path.join(apps_dirpath, '0', '.keep'), 'w') as fd:
fd.write('.')
# Now, copy content from private_storage to that temp directory. We don't
# need to worry about creating the directories locally, the storage class
# does that for us.
for dirpath, dirnames, filenames in walk_storage(
apps_dirpath, storage=private_storage):
for filename in filenames:
src_path = os.path.join(dirpath, filename)
dst_path = os.path.join(
local_source_dir, 'apps', os.path.basename(dirpath), filename)
copy_stored_file(
src_path, dst_path, src_storage=private_storage,
dst_storage=local_storage)
# Also add extra files to the temp directory.
extra_filenames = compile_extra_files(local_source_dir, date)
# All our files are now present locally, let's generate a local filename
# that will contain the final '.tar.gz' before it's copied over to
# public storage.
local_target_file = tempfile.NamedTemporaryFile(
suffix='.tgz', prefix='dumped-apps-')
# tar ALL the things!
cmd = ['tar', 'czf', local_target_file.name, '-C',
local_source_dir] + ['apps'] + extra_filenames
task_log.info(u'Creating dump {0}'.format(local_target_file.name))
subprocess.call(cmd)
# Now copy the local tgz to the public storage.
remote_target_filename = os.path.join(
settings.DUMPED_APPS_PATH, 'tarballs', '%s.tgz' % tarball_name)
copy_stored_file(local_target_file.name, remote_target_filename,
src_storage=local_storage,
dst_storage=public_storage)
# Clean-up.
local_target_file.close()
rm_directory(local_source_dir)
return remote_target_filename
def resize_video(src, pk, user_pk=None, **kw):
"""Try and resize a video and cope if it fails."""
instance = Preview.objects.get(pk=pk)
user = UserProfile.objects.get(pk=user_pk) if user_pk else None
try:
copy_stored_file(src, src, src_storage=private_storage,
dst_storage=local_storage)
result = _resize_video(src, instance, **kw)
except Exception, err:
log.error('Error on processing video: %s' % err)
_resize_error(src, instance, user)
raise
def test_disabled_but_admin(self):
self.login("*****@*****.**")
self.file.update(status=mkt.STATUS_DISABLED)
copy_stored_file(
self.packaged_app_path("mozball.zip"),
self.file.file_path,
src_storage=local_storage,
dst_storage=private_storage,
)
path = private_storage.url(self.file.file_path)
res = self.client.get(self.url)
self.assert3xx(res, path)
def resize_video(src, pk, user_pk=None, **kw):
"""Try and resize a video and cope if it fails."""
instance = Preview.objects.get(pk=pk)
user = UserProfile.objects.get(pk=user_pk) if user_pk else None
try:
copy_stored_file(src, src, src_storage=private_storage,
dst_storage=local_storage)
result = _resize_video(src, instance, **kw)
except Exception, err:
log.error('Error on processing video: %s' % err)
_resize_error(src, instance, user)
raise
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name, 'position': 1})
copy_stored_file(
get_image_path(name), os.path.join(self.dest, name),
src_storage=local_storage, dst_storage=private_storage)
assert form.is_valid(), form.errors
form.save(self.addon)
# Since the task is a post-request-task and we are outside the normal
# request-response cycle, manually send the tasks.
post_request_task._send_tasks()
eq_(self.addon.previews.all()[0].sizes,
{u'image': [250, 297], u'thumbnail': [100, 119]})
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name, 'position': 1})
copy_stored_file(get_image_path(name),
os.path.join(self.dest, name),
src_storage=local_storage,
dst_storage=private_storage)
assert form.is_valid(), form.errors
form.save(self.addon)
eq_(self.addon.previews.all()[0].sizes, {
u'image': [250, 297],
u'thumbnail': [100, 119]
})
def test_parse_packaged_BOM(self):
path = self.packaged_app_path('mozBOM.zip')
if storage_is_remote():
copy_stored_file(path, path, src_storage=local_storage,
dst_storage=private_storage)
wp = WebAppParser().parse(private_storage.open(path))
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozBOM ょ')
eq_(wp['description']['en-US'], u'Exciting BOM action!')
eq_(wp['description']['es'], u'¡Acción BOM!')
eq_(wp['description']['it'], u'Azione BOM!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def setUp(self):
self.img_path = tempfile.mktemp()
copy_stored_file(get_image_path('mozilla.png'),
self.img_path,
src_storage=local_storage,
dest_storage=public_storage)
patcher = mock.patch('subprocess.Popen')
self.mock_popen = patcher.start()
attrs = {
'returncode': 0,
'communicate.return_value': ('ouput', 'error')
}
self.mock_popen.return_value.configure_mock(**attrs)
self.addCleanup(patcher.stop)
def setUp(self):
self.img_path = tempfile.mktemp()
copy_stored_file(get_image_path('mozilla.png'),
self.img_path,
src_storage=local_storage,
dest_storage=public_storage)
patcher = mock.patch('subprocess.Popen')
self.mock_popen = patcher.start()
attrs = {
'returncode': 0,
'communicate.return_value': ('ouput', 'error')
}
self.mock_popen.return_value.configure_mock(**attrs)
self.addCleanup(patcher.stop)
def test_parse_packaged_BOM(self):
path = self.packaged_app_path('mozBOM.zip')
if storage_is_remote():
copy_stored_file(path,
path,
src_storage=local_storage,
dst_storage=private_storage)
wp = WebAppParser().parse(private_storage.open(path))
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozBOM ょ')
eq_(wp['description']['en-US'], u'Exciting BOM action!')
eq_(wp['description']['es'], u'¡Acción BOM!')
eq_(wp['description']['it'], u'Azione BOM!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def test_resize_transparency():
src = get_image_path('transparent.png')
dest = tempfile.mkstemp(dir=settings.TMP_PATH)[1]
expected = src.replace('.png', '-expected.png')
if storage_is_remote():
copy_stored_file(src, src, src_storage=local_storage,
dst_storage=private_storage)
try:
resize_image(src, dest, (32, 32), remove_src=False)
with public_storage.open(dest) as dfh:
with open(expected) as efh:
assert dfh.read() == efh.read()
finally:
if public_storage.exists(dest):
public_storage.delete(dest)
def test_set_modified(self, mock_move, update_mock):
"""Test passed instance is updated with the hash."""
expected_suffix = '.opti.png'
tmp_src = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
tmp_dest = os.path.splitext(tmp_src.name)[0] + expected_suffix
copy_stored_file(self.img_path, tmp_dest, src_storage=public_storage,
dst_storage=local_storage)
with mock.patch('tempfile.NamedTemporaryFile',
lambda *a, **k: tmp_src):
obj = app_factory()
ret = tasks.pngcrush_image(self.img_path, 'some_hash',
set_modified_on=[obj])
ok_('some_hash' in ret)
eq_(update_mock.call_args_list[-1][1]['some_hash'], ret['some_hash'])
ok_('modified' in update_mock.call_args_list[-1][1])
def setUp(self):
super(TestTask, self).setUp()
self.app = Webapp.objects.get(pk=337141)
self.preview = Preview.objects.create(
addon=self.app, thumbnail_path=tempfile.mkstemp()[1],
image_path=tempfile.mkstemp()[1])
# Copy files to private storage where `resize_video` expects it.
self.tmp_good = tempfile.NamedTemporaryFile(suffix='.webm').name
self.tmp_bad = tempfile.NamedTemporaryFile(suffix='.png').name
copy_stored_file(files['good'], self.tmp_good,
src_storage=local_storage,
dst_storage=private_storage)
copy_stored_file(files['bad'], self.tmp_bad,
src_storage=local_storage,
dst_storage=private_storage)
def pngcrush_image(src, hash_field='image_hash', storage=public_storage, **kw):
"""
Optimizes a PNG image by running it through Pngcrush. Returns hash.
src -- filesystem image path
hash_field -- field name to save the new hash on instance if passing
instance through set_modified_on
"""
log.info('[1@None] Optimizing image: %s' % src)
tmp_src = tempfile.NamedTemporaryFile(suffix='.png')
with storage.open(src) as srcf:
shutil.copyfileobj(srcf, tmp_src)
tmp_src.seek(0)
try:
# pngcrush -ow has some issues, use a temporary file and do the final
# renaming ourselves.
suffix = '.opti.png'
tmp_path = '%s%s' % (os.path.splitext(tmp_src.name)[0], suffix)
cmd = [
settings.PNGCRUSH_BIN, '-q', '-rem', 'alla', '-brute', '-reduce',
'-e', suffix, tmp_src.name
]
sp = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = sp.communicate()
if sp.returncode != 0:
log.error('Error optimizing image: %s; %s' % (src, stderr.strip()))
kw['storage'] = storage
pngcrush_image.retry(args=[src], kwargs=kw, max_retries=3)
return False
# Return hash for set_modified_on.
with open(tmp_path) as fd:
image_hash = _hash_file(fd)
copy_stored_file(tmp_path,
src,
src_storage=local_storage,
dst_storage=storage)
log.info('Image optimization completed for: %s' % src)
os.remove(tmp_path)
tmp_src.close()
return {hash_field: image_hash}
except Exception, e:
log.error('Error optimizing image: %s; %s' % (src, e))
def handle_file_operations(self, upload):
"""Copy the file attached to a FileUpload to the Extension instance."""
upload.path = smart_path(nfd_str(upload.path))
if private_storage.exists(self.file_path):
# The filename should not exist. If it does, it means we are trying
# to re-upload the same version. This should have been caught
# before, so just raise an exception.
raise RuntimeError(
'Trying to upload a file to a destination that already exists')
# Copy file from fileupload. This uses private_storage for now as the
# unreviewed, unsigned filename is private.
copy_stored_file(
upload.path, self.file_path,
src_storage=private_storage, dst_storage=private_storage)
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name, 'position': 1})
copy_stored_file(get_image_path(name),
os.path.join(self.dest, name),
src_storage=local_storage,
dst_storage=private_storage)
assert form.is_valid(), form.errors
form.save(self.addon)
# Since the task is a post-request-task and we are outside the normal
# request-response cycle, manually send the tasks.
post_request_task._send_tasks()
eq_(self.addon.previews.all()[0].sizes, {
u'image': [250, 297],
u'thumbnail': [100, 119]
})
def test_parse_packaged(self):
path = self.packaged_app_path('mozball.zip')
if storage_is_remote():
copy_stored_file(path, path, src_storage=local_storage,
dst_storage=private_storage)
wp = WebAppParser().parse(private_storage.open(path))
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozillaBall ょ')
eq_(wp['description']['en-US'],
u'Exciting Open Web development action!')
eq_(wp['description']['es'],
u'¡Acción abierta emocionante del desarrollo del Web!')
eq_(wp['description']['it'],
u'Azione aperta emozionante di sviluppo di fotoricettore!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def setup_files(self, filename='mozball.zip'):
# Local source filename must exist.
assert os.path.exists(self.packaged_app_path(filename))
# Remote filename must not be empty.
assert self.file.filename
# Original packaged file.
copy_stored_file(self.packaged_app_path(filename),
self.file.file_path,
src_storage=local_storage,
dst_storage=private_storage)
# Signed packaged file.
copy_stored_file(self.packaged_app_path(filename),
self.file.signed_file_path,
src_storage=local_storage,
dst_storage=public_storage)
def setup_files(self, filename='mozball.zip'):
# Local source filename must exist.
assert os.path.exists(self.packaged_app_path(filename))
# Remote filename must not be empty.
assert self.file.filename
# Original packaged file.
copy_stored_file(self.packaged_app_path(filename),
self.file.file_path,
src_storage=local_storage,
dst_storage=private_storage)
# Signed packaged file.
copy_stored_file(self.packaged_app_path(filename),
self.file.signed_file_path,
src_storage=local_storage,
dst_storage=public_storage)
def test_parse_packaged(self):
path = self.packaged_app_path('mozball.zip')
if storage_is_remote():
copy_stored_file(path,
path,
src_storage=local_storage,
dst_storage=private_storage)
wp = WebAppParser().parse(private_storage.open(path))
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozillaBall ょ')
eq_(wp['description']['en-US'],
u'Exciting Open Web development action!')
eq_(wp['description']['es'],
u'¡Acción abierta emocionante del desarrollo del Web!')
eq_(wp['description']['it'],
u'Azione aperta emozionante di sviluppo di fotoricettore!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def mock_sign(version_id, reviewer=False):
"""
This is a mock for using in tests, where we really don't want to be
actually signing the apps. This just copies the file over and returns
the path. It doesn't have much error checking.
"""
version = Version.objects.get(pk=version_id)
file_obj = version.all_files[0]
if reviewer:
path = file_obj.signed_reviewer_file_path
storage = private_storage
else:
path = file_obj.signed_file_path
storage = public_storage
copy_stored_file(
file_obj.file_path, path,
src_storage=private_storage, dst_storage=storage)
return path
def setUp(self):
super(TestTask, self).setUp()
self.app = Webapp.objects.get(pk=337141)
self.preview = Preview.objects.create(
addon=self.app,
thumbnail_path=tempfile.mkstemp()[1],
image_path=tempfile.mkstemp()[1])
# Copy files to private storage where `resize_video` expects it.
self.tmp_good = tempfile.NamedTemporaryFile(suffix='.webm').name
self.tmp_bad = tempfile.NamedTemporaryFile(suffix='.png').name
copy_stored_file(files['good'],
self.tmp_good,
src_storage=local_storage,
dst_storage=private_storage)
copy_stored_file(files['bad'],
self.tmp_bad,
src_storage=local_storage,
dst_storage=private_storage)
def handle_file_operations(self, upload):
"""Copy the file attached to a FileUpload to the Extension instance."""
upload.path = smart_path(nfd_str(upload.path))
if not self.slug:
raise RuntimeError(
'Trying to upload a file belonging to a slugless extension')
if private_storage.exists(self.file_path):
# The filename should not exist. If it does, it means we are trying
# to re-upload the same version. This should have been caught
# before, so just raise an exception.
raise RuntimeError(
'Trying to upload a file to a destination that already exists')
# Copy file from fileupload. This uses private_storage for now as the
# unreviewed, unsigned filename is private.
copy_stored_file(upload.path, self.file_path)
def pngcrush_image(src, hash_field='image_hash', storage=public_storage, **kw):
"""
Optimizes a PNG image by running it through Pngcrush. Returns hash.
src -- filesystem image path
hash_field -- field name to save the new hash on instance if passing
instance through set_modified_on
"""
log.info('[1@None] Optimizing image: %s' % src)
tmp_src = tempfile.NamedTemporaryFile(suffix='.png')
with storage.open(src) as srcf:
shutil.copyfileobj(srcf, tmp_src)
tmp_src.seek(0)
try:
# pngcrush -ow has some issues, use a temporary file and do the final
# renaming ourselves.
suffix = '.opti.png'
tmp_path = '%s%s' % (os.path.splitext(tmp_src.name)[0], suffix)
cmd = [settings.PNGCRUSH_BIN, '-q', '-rem', 'alla', '-brute',
'-reduce', '-e', suffix, tmp_src.name]
sp = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = sp.communicate()
if sp.returncode != 0:
log.error('Error optimizing image: %s; %s' % (src, stderr.strip()))
kw['storage'] = storage
pngcrush_image.retry(args=[src], kwargs=kw, max_retries=3)
return False
# Return hash for set_modified_on.
with open(tmp_path) as fd:
image_hash = _hash_file(fd)
copy_stored_file(tmp_path, src, src_storage=local_storage,
dst_storage=storage)
log.info('Image optimization completed for: %s' % src)
os.remove(tmp_path)
tmp_src.close()
return {
hash_field: image_hash
}
except Exception, e:
log.error('Error optimizing image: %s; %s' % (src, e))
def test_admin_can_blocklist(self):
blocklist_zip_path = os.path.join(settings.MEDIA_ROOT,
'packaged-apps', 'blocklisted.zip')
if storage_is_remote():
copy_stored_file(blocklist_zip_path, blocklist_zip_path,
src_storage=local_storage,
dst_storage=private_storage)
self.grant_permission(
UserProfile.objects.get(email='*****@*****.**'),
'Apps:Configure')
self.login('*****@*****.**')
v_count = self.app.versions.count()
url = self.app.get_dev_url('blocklist')
res = self.client.post(url)
self.assert3xx(res, self.app.get_dev_url('versions'))
app = self.app.reload()
eq_(app.versions.count(), v_count + 1)
eq_(app.status, mkt.STATUS_BLOCKED)
eq_(app.versions.latest().files.latest().status, mkt.STATUS_BLOCKED)
def upload(self, name, **kwargs):
if os.path.splitext(name)[-1] not in ['.webapp', '.zip']:
name = name + '.zip'
v = json.dumps(dict(errors=0, warnings=1, notices=2, metadata={}))
fname = nfd_str(self.packaged_app_path(name))
if not local_storage.exists(fname):
raise ValueError('The file %s does not exist :(', fname)
if not private_storage.exists(fname):
copy_stored_file(fname, fname, src_storage=local_storage,
dst_storage=private_storage)
data = {
'path': fname,
'name': name,
'hash': 'sha256:%s' % name,
'validation': v
}
data.update(**kwargs)
return FileUpload.objects.create(**data)
def test_admin_can_blocklist(self):
blocklist_zip_path = os.path.join(settings.MEDIA_ROOT,
'packaged-apps', 'blocklisted.zip')
if storage_is_remote():
copy_stored_file(blocklist_zip_path, blocklist_zip_path,
src_storage=local_storage,
dst_storage=private_storage)
self.grant_permission(
UserProfile.objects.get(email='*****@*****.**'),
'Apps:Configure')
self.login('*****@*****.**')
v_count = self.app.versions.count()
url = self.app.get_dev_url('blocklist')
res = self.client.post(url)
self.assert3xx(res, self.app.get_dev_url('versions'))
app = self.app.reload()
eq_(app.versions.count(), v_count + 1)
eq_(app.status, mkt.STATUS_BLOCKED)
eq_(app.versions.latest().files.latest().status, mkt.STATUS_BLOCKED)
def mock_sign(version_id, reviewer=False):
"""
This is a mock for using in tests, where we really don't want to be
actually signing the apps. This just copies the file over and returns
the path. It doesn't have much error checking.
"""
version = Version.objects.get(pk=version_id)
file_obj = version.all_files[0]
if reviewer:
path = file_obj.signed_reviewer_file_path
storage = private_storage
else:
path = file_obj.signed_file_path
storage = public_storage
copy_stored_file(file_obj.file_path,
path,
src_storage=private_storage,
dst_storage=storage)
return path
def from_upload(cls, upload, version, parse_data={}):
upload.path = amo.utils.smart_path(nfd_str(upload.path))
ext = os.path.splitext(upload.path)[1]
f = cls(version=version)
f.filename = f.generate_filename(extension=ext or '.zip')
f.size = storage.size(upload.path) # Size in bytes.
f.status = amo.STATUS_PENDING
f.hash = f.generate_hash(upload.path)
f.save()
log.debug('New file: %r from %r' % (f, upload))
# Move the uploaded file from the temp location.
copy_stored_file(upload.path, os.path.join(version.path_prefix,
nfd_str(f.filename)))
if upload.validation:
FileValidation.from_json(f, upload.validation)
return f
def upload(self, name, **kwargs):
if os.path.splitext(name)[-1] not in ['.webapp', '.zip']:
name = name + '.zip'
v = json.dumps(dict(errors=0, warnings=1, notices=2, metadata={}))
fname = nfd_str(self.packaged_app_path(name))
if not local_storage.exists(fname):
raise ValueError('The file %s does not exist :(', fname)
if not private_storage.exists(fname):
copy_stored_file(fname,
fname,
src_storage=local_storage,
dst_storage=private_storage)
data = {
'path': fname,
'name': name,
'hash': 'sha256:%s' % name,
'validation': v
}
data.update(**kwargs)
return FileUpload.objects.create(**data)