def run_validator(file_path, url=None):
"""A pre-configured wrapper around the app validator."""
temp_path = None
# Make a copy of the file since we can't assume the
# uploaded file is on the local filesystem.
temp_path = tempfile.mktemp()
with open(temp_path, "wb") as local_f:
with private_storage.open(file_path) as remote_f:
copyfileobj(remote_f, local_f)
with statsd.timer("mkt.developers.validator"):
is_packaged = zipfile.is_zipfile(temp_path)
if is_packaged:
log.info(u"Running `validate_packaged_app` for path: %s" % (file_path))
with statsd.timer("mkt.developers.validate_packaged_app"):
return validate_packaged_app(
temp_path,
market_urls=settings.VALIDATOR_IAF_URLS,
timeout=settings.VALIDATOR_TIMEOUT,
spidermonkey=settings.SPIDERMONKEY,
)
else:
log.info(u"Running `validate_app` for path: %s" % (file_path))
with statsd.timer("mkt.developers.validate_app"):
return validate_app(open(temp_path).read(), market_urls=settings.VALIDATOR_IAF_URLS, url=url)
# Clean up copied files.
os.unlink(temp_path)
def test_preview_size(self):
name = "non-animated.gif"
form = forms.PreviewForm({"upload_hash": name, "position": 1})
with storage.open(os.path.join(self.dest, name), "wb") as f:
copyfileobj(open(get_image_path(name)), f)
assert form.is_valid(), form.errors
form.save(self.addon)
eq_(self.addon.previews.all()[0].sizes, {u"image": [250, 297], u"thumbnail": [100, 119]})
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name,
'position': 1})
with storage.open(os.path.join(self.dest, name), 'wb') as f:
copyfileobj(open(get_image_path(name)), f)
assert form.is_valid(), form.errors
form.save(self.addon)
eq_(self.addon.previews.all()[0].sizes,
{u'image': [250, 297], u'thumbnail': [100, 119]})
def setUp(self):
self.webapp_path = tempfile.mktemp(suffix='.webapp')
with storage.open(self.webapp_path, 'wb') as f:
copyfileobj(open(os.path.join(os.path.dirname(__file__),
'addons', 'mozball.webapp')),
f)
self.tmp_files = []
self.manifest = dict(name=u'Ivan Krsti\u0107', version=u'1.0',
description=u'summary',
developer=dict(name=u'Dev Namé'))
def setUp(self):
self.webapp_path = tempfile.mktemp(suffix='.webapp')
with storage.open(self.webapp_path, 'wb') as f:
copyfileobj(open(os.path.join(os.path.dirname(__file__),
'addons', 'mozball.webapp')),
f)
self.tmp_files = []
self.manifest = dict(name=u'Ivan Krsti\u0107', version=u'1.0',
description=u'summary',
developer=dict(name=u'Dev Namé'))
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name,
'position': 1})
with storage.open(os.path.join(self.dest, name), 'wb') as f:
copyfileobj(open(get_image_path(name)), f)
assert form.is_valid(), form.errors
form.save(self.addon)
eq_(self.addon.previews.all()[0].sizes,
{u'image': [250, 297], u'thumbnail': [100, 119]})
def upload(self, name):
if os.path.splitext(name)[-1] not in ['.webapp', '.zip']:
name = name + '.zip'
v = json.dumps(dict(errors=0, warnings=1, notices=2, metadata={}))
fname = nfd_str(self.packaged_app_path(name))
if not storage.exists(fname):
with storage.open(fname, 'w') as fs:
copyfileobj(open(fname), fs)
d = dict(path=fname, name=name, hash='sha256:%s' % name, validation=v)
return FileUpload.objects.create(**d)
def test_no_manifest_at_root(self):
path = self.packaged_app_path('no-manifest-at-root.zip')
if storage_is_remote():
with open(path) as local_f:
with storage.open(path, 'w') as remote_f:
copyfileobj(local_f, remote_f)
with self.assertRaises(forms.ValidationError) as exc:
WebAppParser().parse(path)
m = exc.exception.messages[0]
assert m.startswith('The file "manifest.webapp" was not found'), (
'Unexpected: %s' % m)
def test_no_manifest_at_root(self):
path = self.packaged_app_path('no-manifest-at-root.zip')
if storage_is_remote():
with open(path) as local_f:
with storage.open(path, 'w') as remote_f:
copyfileobj(local_f, remote_f)
with self.assertRaises(forms.ValidationError) as exc:
WebAppParser().parse(path)
m = exc.exception.messages[0]
assert m.startswith('The file "manifest.webapp" was not found'), (
'Unexpected: %s' % m)
def upload(self, name):
if os.path.splitext(name)[-1] not in ['.webapp', '.zip']:
name = name + '.zip'
v = json.dumps(dict(errors=0, warnings=1, notices=2, metadata={}))
fname = nfd_str(self.packaged_app_path(name))
if not storage.exists(fname):
with storage.open(fname, 'w') as fs:
copyfileobj(open(fname), fs)
d = dict(path=fname, name=name,
hash='sha256:%s' % name, validation=v)
return FileUpload.objects.create(**d)
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name, 'position': 1})
with private_storage.open(os.path.join(self.dest, name), 'wb') as f:
copyfileobj(open(get_image_path(name)), f)
assert form.is_valid(), form.errors
form.save(self.addon)
# Since the task is a post-request-task and we are outside the normal
# request-response cycle, manually send the tasks.
post_request_task._send_tasks()
eq_(self.addon.previews.all()[0].sizes,
{u'image': [250, 297], u'thumbnail': [100, 119]})
def test_icon_modified(self, update_mock):
name = 'transparent.png'
form = forms.AddonFormMedia({'icon_upload_hash': name},
request=self.request,
instance=self.addon)
dest = os.path.join(self.icon_path, name)
with storage.open(dest, 'w') as f:
copyfileobj(open(get_image_path(name)), f)
assert form.is_valid()
form.save(addon=self.addon)
assert update_mock.called
def upload(self, name, **kwargs):
if os.path.splitext(name)[-1] not in [".webapp", ".zip"]:
name = name + ".zip"
v = json.dumps(dict(errors=0, warnings=1, notices=2, metadata={}))
fname = nfd_str(self.packaged_app_path(name))
if not storage.exists(fname):
with storage.open(fname, "w") as fs:
copyfileobj(open(fname), fs)
data = {"path": fname, "name": name, "hash": "sha256:%s" % name, "validation": v}
data.update(**kwargs)
return FileUpload.objects.create(**data)
def validator(upload_id, **kw):
if not settings.VALIDATE_ADDONS:
return None
log.info(u'[FileUpload:%s] Validating app.' % upload_id)
try:
upload = FileUpload.objects.get(pk=upload_id)
except FileUpload.DoesNotExist:
log.info(u'[FileUpload:%s] Does not exist.' % upload_id)
return
temp_path = None
# Make a copy of the file since we can't assume the
# uploaded file is on the local filesystem.
temp_path = tempfile.mktemp()
with open(temp_path, 'wb') as local_f:
with storage.open(upload.path) as remote_f:
copyfileobj(remote_f, local_f)
try:
validation_result = run_validator(temp_path, url=kw.get('url'))
if upload.validation:
# If there's any preliminary validation result, merge it with the
# actual validation result.
dec_prelim_result = json.loads(upload.validation)
if 'prelim' in dec_prelim_result:
dec_validation_result = json.loads(validation_result)
# Merge the messages.
dec_validation_result['messages'] += (
dec_prelim_result['messages'])
# Merge the success value.
if dec_validation_result['success']:
dec_validation_result['success'] = (
dec_prelim_result['success'])
# Merge the error count (we only raise errors, not warnings).
dec_validation_result['errors'] += dec_prelim_result['errors']
# Put the validation result back into JSON.
validation_result = json.dumps(dec_validation_result)
upload.validation = validation_result
upload.save() # We want to hit the custom save().
except Exception:
# Store the error with the FileUpload job, then raise
# it for normal logging.
tb = traceback.format_exception(*sys.exc_info())
upload.update(task_error=''.join(tb))
# Don't raise if we're being eager, setting the error is enough.
if not settings.CELERY_ALWAYS_EAGER:
raise
# Clean up copied files.
os.unlink(temp_path)
def test_parse_packaged_BOM(self):
path = self.packaged_app_path('mozBOM.zip')
if storage_is_remote():
with open(path) as local_f:
with storage.open(path, 'w') as remote_f:
copyfileobj(local_f, remote_f)
wp = WebAppParser().parse(path)
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozBOM ょ')
eq_(wp['description']['en-US'], u'Exciting BOM action!')
eq_(wp['description']['es'], u'¡Acción BOM!')
eq_(wp['description']['it'], u'Azione BOM!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def test_parse_packaged_BOM(self):
path = self.packaged_app_path('mozBOM.zip')
if storage_is_remote():
with open(path) as local_f:
with storage.open(path, 'w') as remote_f:
copyfileobj(local_f, remote_f)
wp = WebAppParser().parse(path)
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozBOM ょ')
eq_(wp['description']['en-US'], u'Exciting BOM action!')
eq_(wp['description']['es'], u'¡Acción BOM!')
eq_(wp['description']['it'], u'Azione BOM!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def resize_video(src, pk, user_pk=None, **kw):
"""Try and resize a video and cope if it fails."""
instance = Preview.objects.get(pk=pk)
user = UserProfile.objects.get(pk=user_pk) if user_pk else None
try:
if not os.path.exists(src):
# We're probably using S3 in this case.
try:
os.makedirs(os.path.dirname(src))
except OSError: # already exists
pass
copyfileobj(public_storage.open(src), open(src, 'w'))
result = _resize_video(src, instance, **kw)
except Exception, err:
log.error('Error on processing video: %s' % err)
_resize_error(src, instance, user)
raise
def test_parse_packaged(self):
path = self.packaged_app_path('mozball.zip')
if storage_is_remote():
with open(path) as local_f:
with storage.open(path, 'w') as remote_f:
copyfileobj(local_f, remote_f)
wp = WebAppParser().parse(path)
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozillaBall ょ')
eq_(wp['description']['en-US'],
u'Exciting Open Web development action!')
eq_(wp['description']['es'],
u'¡Acción abierta emocionante del desarrollo del Web!')
eq_(wp['description']['it'],
u'Azione aperta emozionante di sviluppo di fotoricettore!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def test_parse_packaged(self):
path = self.packaged_app_path('mozball.zip')
if storage_is_remote():
with open(path) as local_f:
with storage.open(path, 'w') as remote_f:
copyfileobj(local_f, remote_f)
wp = WebAppParser().parse(path)
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozillaBall ょ')
eq_(wp['description']['en-US'],
u'Exciting Open Web development action!')
eq_(wp['description']['es'],
u'¡Acción abierta emocionante del desarrollo del Web!')
eq_(wp['description']['it'],
u'Azione aperta emozionante di sviluppo di fotoricettore!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def resize_video(src, pk, user_pk=None, **kw):
"""Try and resize a video and cope if it fails."""
instance = Preview.objects.get(pk=pk)
user = UserProfile.objects.get(pk=user_pk) if user_pk else None
try:
if not os.path.exists(src):
# We're probably using S3 in this case.
try:
os.makedirs(os.path.dirname(src))
except OSError: # already exists
pass
copyfileobj(public_storage.open(src), open(src, 'w'))
result = _resize_video(src, instance, **kw)
except Exception, err:
log.error('Error on processing video: %s' % err)
_resize_error(src, instance, user)
raise
def packaged_copy_over(self, dest, name):
with storage.open(dest, 'wb') as f:
copyfileobj(open(self.packaged_app_path(name)), f)
def manifest_copy_over(self, dest, name):
with storage.open(dest, 'wb') as f:
copyfileobj(open(self.manifest_path(name)), f)
def manifest_copy_over(self, dest, name):
with storage.open(dest, 'wb') as f:
copyfileobj(open(self.manifest_path(name)), f)
def packaged_copy_over(self, dest, name):
with storage.open(dest, 'wb') as f:
copyfileobj(open(self.packaged_app_path(name)), f)