def generate_packaged_app(namedict, apptype, categories, developer_name,
privacy_policy=None, device_types=(),
permissions=(), versions=(),
default_locale='en-US', package_file=None,
status=4, **kw):
now = datetime.datetime.now()
app = app_factory(categories=categories, name=namedict[default_locale],
complete=False, rated=True, is_packaged=True,
privacy_policy=privacy_policy,
version_kw={
'version': '1.0',
'reviewed': now if status >= 4 else None,
'_developer_name': developer_name},
file_kw={'status': status})
if device_types:
for dt in device_types:
app.addondevicetype_set.create(device_type=DEVICE_CHOICES_IDS[dt])
else:
app.addondevicetype_set.create(device_type=1)
f = app.latest_version.all_files[0]
f.update(filename=f.generate_filename())
fp = os.path.join(app.latest_version.path_prefix, f.filename)
if package_file:
package_file_file = open(package_file)
manifest = WebAppParser().get_json_data(package_file_file)
AppManifest.objects.create(
version=app.latest_version, manifest=json.dumps(manifest))
# copy package_file to storage like a normal app.
private_storage.save(fp, package_file_file)
app.update_version()
return app
with private_storage.open(fp, 'w') as out:
generate_app_package(app, out, apptype,
permissions, namedict,
version=app.latest_version)
for i, vspec in enumerate(versions, 1):
st = STATUS_CHOICES_API_LOOKUP[vspec.get("status", "public")]
rtime = (now + datetime.timedelta(i))
v = version_factory(version="1." + str(i), addon=app,
reviewed=rtime if st >= 4 else None,
nomination=rtime if st > 0 else None,
created=rtime,
file_kw={'status': st},
_developer_name=developer_name)
f = v.files.all()[0]
f.update(filename=f.generate_filename())
fp = os.path.join(app.latest_version.path_prefix, f.filename)
with private_storage.open(fp, 'w') as out:
generate_app_package(app, out, vspec.get("type", apptype),
vspec.get("permissions", permissions),
namedict, version=v)
app.update_version()
return app
def generate_packaged_app(namedict, apptype, categories, developer_name,
privacy_policy=None, device_types=(),
permissions=(), versions=(),
default_locale='en-US', package_file=None,
status=4, **kw):
now = datetime.datetime.now()
app = app_factory(categories=categories, name=namedict[default_locale],
complete=False, rated=True, is_packaged=True,
privacy_policy=privacy_policy,
version_kw={
'version': '1.0',
'reviewed': now if status >= 4 else None,
'_developer_name': developer_name},
file_kw={'status': status})
if device_types:
for dt in device_types:
app.addondevicetype_set.create(device_type=DEVICE_CHOICES_IDS[dt])
else:
app.addondevicetype_set.create(device_type=1)
f = app.latest_version.all_files[0]
f.update(filename=f.generate_filename())
fp = os.path.join(app.latest_version.path_prefix, f.filename)
if package_file:
package_file_file = open(package_file)
manifest = WebAppParser().get_json_data(package_file_file)
AppManifest.objects.create(
version=app.latest_version, manifest=json.dumps(manifest))
# copy package_file to storage like a normal app.
private_storage.save(fp, package_file_file)
app.update_version()
return app
with private_storage.open(fp, 'w') as out:
generate_app_package(app, out, apptype,
permissions, namedict,
version=app.latest_version)
for i, vspec in enumerate(versions, 1):
st = STATUS_CHOICES_API_LOOKUP[vspec.get("status", "public")]
rtime = (now + datetime.timedelta(i))
v = version_factory(version="1." + str(i), addon=app,
reviewed=rtime if st >= 4 else None,
nomination=rtime if st > 0 else None,
created=rtime,
file_kw={'status': st},
_developer_name=developer_name)
f = v.files.all()[0]
f.update(filename=f.generate_filename())
fp = os.path.join(app.latest_version.path_prefix, f.filename)
with private_storage.open(fp, 'w') as out:
generate_app_package(app, out, vspec.get("type", apptype),
vspec.get("permissions", permissions),
namedict, version=v)
app.update_version()
return app
def mock_sign(version_id, reviewer=False):
"""
This is a mock for using in tests, where we really don't want to be
actually signing the apps. This just copies the file over and returns
the path. It doesn't have much error checking.
"""
version = Version.objects.get(pk=version_id)
file_obj = version.all_files[0]
path = (file_obj.signed_reviewer_file_path if reviewer else
file_obj.signed_file_path)
with private_storage.open(path, 'w') as dest_f:
shutil.copyfileobj(private_storage.open(file_obj.file_path), dest_f)
return path
def reviewer_sign_file(self):
"""Sign the original file (`file_path`) with reviewer certs, then move
the signed file to the reviewers-specific signed path
(`reviewer_signed_file_path`) on private storage."""
if not self.extension.uuid:
raise SigningError('Need uuid to be set to sign')
if not self.pk:
raise SigningError('Need version pk to be set to sign')
ids = json.dumps({
# Reviewers get a unique 'id' so the reviewer installed add-on
# won't conflict with the public add-on, and also so even multiple
# versions of the same add-on can be installed side by side with
# other versions.
'id': 'reviewer-{guid}-{version_id}'.format(
guid=self.extension.uuid, version_id=self.pk),
'version': self.pk
})
with statsd.timer('extensions.sign_reviewer'):
try:
# This will read the file from self.file_path, generate a
# reviewer signature and write the signed file to
# self.reviewer_signed_file_path.
sign_app(private_storage.open(self.file_path),
self.reviewer_signed_file_path, ids, reviewer=True)
except SigningError:
log.info(
'[ExtensionVersion:%s] Reviewer Signing failed' % self.pk)
if private_storage.exists(self.reviewer_signed_file_path):
private_storage.delete(self.reviewer_signed_file_path)
raise
def sign_file(self):
"""Sign the original file (`file_path`), then move signed extension
file to the signed path (`signed_file_path`) on public storage. The
original file remains on private storage.
Return the signed file size."""
if not self.extension.uuid:
raise SigningError('Need uuid to be set to sign')
if not self.pk:
raise SigningError('Need version pk to be set to sign')
ids = json.dumps({
# 'id' needs to be an unique identifier not shared with anything
# else (other extensions, langpacks, webapps...), but should not
# change when there is an update.
'id': self.extension.uuid,
# 'version' should be an integer and should be monotonically
# increasing.
'version': self.pk
})
with statsd.timer('extensions.sign'):
try:
# This will read the file from self.file_path, generate a
# signature and write the signed file to self.signed_file_path.
sign_app(private_storage.open(self.file_path),
self.signed_file_path, ids)
except SigningError:
log.info('[ExtensionVersion:%s] Signing failed' % self.pk)
self.remove_public_signed_file() # Clean up.
raise
return public_storage.size(self.signed_file_path)
def resize_preview(src, pk, **kw):
"""Resizes preview images and stores the sizes on the preview."""
instance = Preview.objects.get(pk=pk)
thumb_dst, full_dst = instance.thumbnail_path, instance.image_path
sizes = instance.sizes or {}
log.info("[1@None] Resizing preview and storing size: %s" % thumb_dst)
try:
thumbnail_size = APP_PREVIEW_SIZES[0][:2]
image_size = APP_PREVIEW_SIZES[1][:2]
with private_storage.open(src, "rb") as fp:
size = Image.open(fp).size
if size[0] > size[1]:
# If the image is wider than tall, then reverse the wanted size
# to keep the original aspect ratio while still resizing to
# the correct dimensions.
thumbnail_size = thumbnail_size[::-1]
image_size = image_size[::-1]
if kw.get("generate_thumbnail", True):
sizes["thumbnail"] = resize_image(src, thumb_dst, thumbnail_size, remove_src=False)
if kw.get("generate_image", True):
sizes["image"] = resize_image(src, full_dst, image_size, remove_src=False)
instance.sizes = sizes
instance.save()
log.info("Preview resized to: %s" % thumb_dst)
# Remove src file now that it has been processed.
private_storage.delete(src)
return True
except Exception, e:
log.error("Error saving preview: %s; %s" % (e, thumb_dst))
def test_view_left_binary(self):
self.file_viewer.extract()
filename = os.path.join(self.file_viewer.left.dest, 'script.js')
with private_storage.open(filename, 'w') as f:
f.write('MZ')
res = self.client.get(self.file_url(not_binary))
assert 'This file is not viewable online' in res.content
def run_validator(file_path, url=None):
"""A pre-configured wrapper around the app validator."""
temp_path = None
# Make a copy of the file since we can't assume the
# uploaded file is on the local filesystem.
temp_path = tempfile.mktemp()
with open(temp_path, "wb") as local_f:
with private_storage.open(file_path) as remote_f:
copyfileobj(remote_f, local_f)
with statsd.timer("mkt.developers.validator"):
is_packaged = zipfile.is_zipfile(temp_path)
if is_packaged:
log.info(u"Running `validate_packaged_app` for path: %s" % (file_path))
with statsd.timer("mkt.developers.validate_packaged_app"):
return validate_packaged_app(
temp_path,
market_urls=settings.VALIDATOR_IAF_URLS,
timeout=settings.VALIDATOR_TIMEOUT,
spidermonkey=settings.SPIDERMONKEY,
)
else:
log.info(u"Running `validate_app` for path: %s" % (file_path))
with statsd.timer("mkt.developers.validate_app"):
return validate_app(open(temp_path).read(), market_urls=settings.VALIDATOR_IAF_URLS, url=url)
# Clean up copied files.
os.unlink(temp_path)
def test_view_left_binary(self):
self.file_viewer.extract()
filename = os.path.join(self.file_viewer.left.dest, 'script.js')
with private_storage.open(filename, 'w') as f:
f.write('MZ')
res = self.client.get(self.file_url(not_binary))
assert 'This file is not viewable online' in res.content
def generate_hash(self, filename=None):
"""Generate a hash for a file."""
hash = hashlib.sha256()
with private_storage.open(filename or self.file_path, 'rb') as obj:
for chunk in iter(lambda: obj.read(1024), ''):
hash.update(chunk)
return 'sha256:%s' % hash.hexdigest()
def sign_file(self):
"""Sign the original file (`file_path`), then move signed extension
file to the signed path (`signed_file_path`) on public storage. The
original file remains on private storage.
Return the signed file size."""
if not self.extension.uuid:
raise SigningError('Need uuid to be set to sign')
if not self.pk:
raise SigningError('Need version pk to be set to sign')
if self.extension.is_blocked():
raise SigningError('Trying to signed a blocked extension')
ids = json.dumps({
# 'id' needs to be an unique identifier not shared with anything
# else (other extensions, langpacks, webapps...), but should not
# change when there is an update.
'id': self.extension.uuid,
# 'version' should be an integer and should be monotonically
# increasing.
'version': self.pk
})
with statsd.timer('extensions.sign'):
try:
# This will read the file from self.file_path, generate a
# signature and write the signed file to self.signed_file_path.
sign_app(private_storage.open(self.file_path),
self.signed_file_path, ids)
except SigningError:
log.info('[ExtensionVersion:%s] Signing failed' % self.pk)
self.remove_public_signed_file() # Clean up.
raise
return public_storage.size(self.signed_file_path)
def reviewer_sign_file(self):
"""Sign the original file (`file_path`) with reviewer certs, then move
the signed file to the reviewers-specific signed path
(`reviewer_signed_file_path`) on private storage."""
if not self.extension.uuid:
raise SigningError('Need uuid to be set to sign')
if not self.pk:
raise SigningError('Need version pk to be set to sign')
ids = json.dumps({
'id': self.review_id,
'version': self.pk
})
with statsd.timer('extensions.sign_reviewer'):
try:
# This will read the file from self.file_path, generate a
# reviewer signature and write the signed file to
# self.reviewer_signed_file_path.
sign_app(private_storage.open(self.file_path),
self.reviewer_signed_file_path, ids, reviewer=True)
except SigningError:
log.info(
'[ExtensionVersion:%s] Reviewer Signing failed' % self.pk)
if private_storage.exists(self.reviewer_signed_file_path):
private_storage.delete(self.reviewer_signed_file_path)
raise
def generate_hash(self, filename=None):
"""Generate a hash for a file."""
hash = hashlib.sha256()
with private_storage.open(filename or self.file_path, 'rb') as obj:
for chunk in iter(lambda: obj.read(1024), ''):
hash.update(chunk)
return 'sha256:%s' % hash.hexdigest()
def test_success_add_file(self, validator_mock):
with self.patch_requests() as ur:
ur.iter_content.return_value = mock.Mock(next=lambda: 'woo')
tasks.fetch_manifest('http://xx.com/manifest.json', self.upload.pk)
upload = FileUpload.objects.get(pk=self.upload.pk)
eq_(upload.name, 'http://xx.com/manifest.json')
eq_(private_storage.open(upload.path).read(), 'woo')
def test_all(self, sign_mock):
v1 = self.app.current_version
v2 = self.app2.current_version
file1 = v1.all_files[0]
file2 = v2.all_files[0]
with private_storage.open(file1.file_path, 'w') as f:
f.write('.')
with private_storage.open(file2.file_path, 'w') as f:
f.write('.')
call_command('sign_apps')
eq_(len(sign_mock.mock_calls), 2)
eq_(os.path.join('/', sign_mock.mock_calls[0][1][0].name),
file1.file_path)
eq_(sign_mock.mock_calls[0][1][1], file1.signed_file_path)
eq_(os.path.join('/', sign_mock.mock_calls[1][1][0].name),
file2.file_path)
eq_(sign_mock.mock_calls[1][1][1], file2.signed_file_path)
def make_packaged_app():
ap = app_factory()
ap.update(is_packaged=True)
f = ap.latest_version.all_files[0]
fp = os.path.join(ap.latest_version.path_prefix, f.filename)
with private_storage.open(fp, 'w') as out:
out.write('.')
return ap
def make_packaged_app():
ap = app_factory()
ap.update(is_packaged=True)
f = ap.latest_version.all_files[0]
fp = os.path.join(ap.latest_version.path_prefix, f.filename)
with private_storage.open(fp, 'w') as out:
out.write('.')
return ap
def create_export(self):
date = datetime.datetime.today().strftime("%Y-%m-%d")
with self.settings(DUMPED_USERS_PATH=self.export_directory):
dump_user_installs_cron()
tarball_path = os.path.join(self.export_directory, "tarballs", date + ".tgz")
self.tarfile_file = private_storage.open(tarball_path)
self.tarfile = tarfile.open(fileobj=self.tarfile_file)
return self.tarfile
def test_file_order(self):
self.viewer.extract()
dest = self.viewer.dest
private_storage.open(os.path.join(dest, 'manifest.webapp'),
'w').close()
subdir = os.path.join(dest, 'chrome')
with private_storage.open(os.path.join(subdir, 'foo'), 'w') as f:
f.write('.')
if not private_storage.exists(subdir):
# Might be on S3, which doesn't have directories (and
# django-storages doesn't support empty files).
with private_storage.open(subdir, 'w') as f:
f.write('.')
cache.clear()
files = self.viewer.get_files().keys()
rt = files.index(u'chrome')
eq_(files[rt:rt + 3], [u'chrome', u'chrome/foo', u'dictionaries'])
def test_bom(self):
dest = os.path.join(settings.TMP_PATH, 'test_bom')
with private_storage.open(dest, 'w') as f:
f.write('foo'.encode('utf-16'))
self.viewer.select('foo')
self.viewer.selected = {'full': dest, 'size': 1}
eq_(self.viewer.read_file(), u'foo')
private_storage.delete(dest)
def test_unhide_disabled_files(self):
f = File.objects.get()
f.status = mkt.STATUS_PUBLIC
with private_storage.open(f.guarded_file_path, 'wb') as fp:
fp.write('some data\n')
f.unhide_disabled_file()
assert public_storage.exists(f.file_path)
assert public_storage.open(f.file_path).size
def test_unhide_disabled_files(self):
f = File.objects.get()
f.status = mkt.STATUS_PUBLIC
with private_storage.open(f.guarded_file_path, 'wb') as fp:
fp.write('some data\n')
f.unhide_disabled_file()
assert public_storage.exists(f.file_path)
assert public_storage.open(f.file_path).size
def test_inject_ids(self, post):
post().status_code = 200
post().content = '{"zigbert.rsa": ""}'
packaged.sign(self.version.pk)
zf = zipfile.ZipFile(private_storage.open(self.file.signed_file_path),
mode='r')
ids_data = zf.read('META-INF/ids.json')
eq_(sorted(json.loads(ids_data).keys()), ['id', 'version'])
def test_all(self, sign_mock):
v1 = self.app.current_version
v2 = self.app2.current_version
file1 = v1.all_files[0]
file2 = v2.all_files[0]
with private_storage.open(file1.file_path, 'w') as f:
f.write('.')
with private_storage.open(file2.file_path, 'w') as f:
f.write('.')
call_command('sign_apps')
eq_(len(sign_mock.mock_calls), 2)
eq_(os.path.join('/', sign_mock.mock_calls[0][1][0].name),
file1.file_path)
eq_(sign_mock.mock_calls[0][1][1], file1.signed_file_path)
eq_(os.path.join('/', sign_mock.mock_calls[1][1][0].name),
file2.file_path)
eq_(sign_mock.mock_calls[1][1][1], file2.signed_file_path)
def test_bom(self):
dest = os.path.join(settings.TMP_PATH, 'test_bom')
with private_storage.open(dest, 'w') as f:
f.write('foo'.encode('utf-16'))
self.viewer.select('foo')
self.viewer.selected = {'full': dest, 'size': 1}
eq_(self.viewer.read_file(), u'foo')
private_storage.delete(dest)
def test_file_order(self):
self.viewer.extract()
dest = self.viewer.dest
private_storage.open(os.path.join(dest, 'manifest.webapp'),
'w').close()
subdir = os.path.join(dest, 'chrome')
with private_storage.open(os.path.join(subdir, 'foo'), 'w') as f:
f.write('.')
if not private_storage.exists(subdir):
# Might be on S3, which doesn't have directories (and
# django-storages doesn't support empty files).
with private_storage.open(subdir, 'w') as f:
f.write('.')
cache.clear()
files = self.viewer.get_files().keys()
rt = files.index(u'chrome')
eq_(files[rt:rt + 3], [u'chrome', u'chrome/foo', u'dictionaries'])
def test_success_add_file(self, validator_mock):
with self.patch_requests() as ur:
ur.iter_content.return_value = mock.Mock(next=lambda: 'woo')
tasks.fetch_manifest('http://xx.com/manifest.json', self.upload.pk)
upload = FileUpload.objects.get(pk=self.upload.pk)
eq_(upload.name, 'http://xx.com/manifest.json')
eq_(private_storage.open(upload.path).read(), 'woo')
def create_export(self):
date = datetime.datetime.today().strftime('%Y-%m-%d')
with self.settings(DUMPED_USERS_PATH=self.export_directory):
dump_user_installs_cron()
tarball_path = os.path.join(self.export_directory, 'tarballs',
date + '.tgz')
self.tarfile_file = private_storage.open(tarball_path)
self.tarfile = tarfile.open(fileobj=self.tarfile_file)
return self.tarfile
def get_file(fileorpath):
"""
Get a file-like object, whether given a FileUpload object or an
UploadedFile.
"""
if hasattr(fileorpath, 'path'): # FileUpload
return private_storage.open(fileorpath.path)
if hasattr(fileorpath, 'name'):
return fileorpath
raise ValueError("not a file or upload")
def _get_hash(filename, block_size=2 ** 20, hash=hashlib.md5):
"""Returns an MD5 hash for a filename."""
f = private_storage.open(filename, 'rb')
hash_ = hash()
while True:
data = f.read(block_size)
if not data:
break
hash_.update(data)
return hash_.hexdigest()
def test_by_webapp(self, sign_mock):
v1 = self.app.current_version
file1 = v1.all_files[0]
with private_storage.open(file1.file_path, 'w') as f:
f.write('.')
call_command('sign_apps', webapps=str(self.app.pk))
eq_(sign_mock.call_count, 1)
eq_(os.path.join('/', sign_mock.mock_calls[0][1][0].name),
file1.file_path)
eq_(sign_mock.mock_calls[0][1][1], file1.signed_file_path)
def get_file(fileorpath):
"""
Get a file-like object, whether given a FileUpload object or an
UploadedFile.
"""
if hasattr(fileorpath, 'path'): # FileUpload
return private_storage.open(fileorpath.path)
if hasattr(fileorpath, 'name'):
return fileorpath
raise ValueError("not a file or upload")
def _get_hash(filename, block_size=2**20, hash=hashlib.md5):
"""Returns an MD5 hash for a filename."""
f = private_storage.open(filename, 'rb')
hash_ = hash()
while True:
data = f.read(block_size)
if not data:
break
hash_.update(data)
return hash_.hexdigest()
def test_no_manifest_at_root(self):
path = self.packaged_app_path('no-manifest-at-root.zip')
if storage_is_remote():
copy_stored_file(path, path, src_storage=local_storage,
dst_storage=private_storage)
with self.assertRaises(forms.ValidationError) as exc:
WebAppParser().parse(private_storage.open(path))
m = exc.exception.messages[0]
assert m.startswith('The file "manifest.webapp" was not found'), (
'Unexpected: %s' % m)
def test_by_webapp(self, sign_mock):
v1 = self.app.current_version
file1 = v1.all_files[0]
with private_storage.open(file1.file_path, 'w') as f:
f.write('.')
call_command('sign_apps', webapps=str(self.app.pk))
eq_(sign_mock.call_count, 1)
eq_(os.path.join('/', sign_mock.mock_calls[0][1][0].name),
file1.file_path)
eq_(sign_mock.mock_calls[0][1][1], file1.signed_file_path)
def compress_export(tarball_name, date):
# We need a temporary directory on the local filesystem that will contain
# all files in order to call `tar`.
local_source_dir = tempfile.mkdtemp()
apps_dirpath = os.path.join(settings.DUMPED_APPS_PATH, 'apps')
# In case apps_dirpath is empty, add a dummy file to make the apps
# directory in the tar archive non-empty. It should not happen in prod, but
# it's nice to have it to prevent the task from failing entirely.
with private_storage.open(os.path.join(apps_dirpath, '0', '.keep'),
'w') as fd:
fd.write('.')
# Now, copy content from private_storage to that temp directory. We don't
# need to worry about creating the directories locally, the storage class
# does that for us.
for dirpath, dirnames, filenames in walk_storage(apps_dirpath,
storage=private_storage):
for filename in filenames:
src_path = os.path.join(dirpath, filename)
dst_path = os.path.join(local_source_dir, 'apps',
os.path.basename(dirpath), filename)
copy_stored_file(src_path,
dst_path,
src_storage=private_storage,
dst_storage=local_storage)
# Also add extra files to the temp directory.
extra_filenames = compile_extra_files(local_source_dir, date)
# All our files are now present locally, let's generate a local filename
# that will contain the final '.tar.gz' before it's copied over to
# public storage.
local_target_file = tempfile.NamedTemporaryFile(suffix='.tgz',
prefix='dumped-apps-')
# tar ALL the things!
cmd = ['tar', 'czf', local_target_file.name, '-C', local_source_dir
] + ['apps'] + extra_filenames
task_log.info(u'Creating dump {0}'.format(local_target_file.name))
subprocess.call(cmd)
# Now copy the local tgz to the public storage.
remote_target_filename = os.path.join(settings.DUMPED_APPS_PATH,
'tarballs', '%s.tgz' % tarball_name)
copy_stored_file(local_target_file.name,
remote_target_filename,
src_storage=local_storage,
dst_storage=public_storage)
# Clean-up.
local_target_file.close()
rm_directory(local_source_dir)
return remote_target_filename
def extract_and_validate_upload(cls, upload):
"""Validate and extract manifest from a FileUpload instance.
Can raise ParseError."""
with private_storage.open(upload.path) as file_obj:
# The file will already have been uploaded at this point, so force
# the content type to make the ExtensionValidator happy. We just
# need to validate the contents.
file_obj.content_type = 'application/zip'
manifest_contents = ExtensionValidator(file_obj).validate()
return manifest_contents
def extract_and_validate_upload(cls, upload):
"""Validate and extract manifest from a FileUpload instance.
Can raise ParseError."""
with private_storage.open(upload.path) as file_obj:
# The file will already have been uploaded at this point, so force
# the content type to make the ExtensionValidator happy. We just
# need to validate the contents.
file_obj.content_type = 'application/zip'
manifest_contents = ExtensionValidator(file_obj).validate()
return manifest_contents
def test_parse(self):
wp = WebAppParser().parse(private_storage.open(self.webapp_path))
eq_(wp['guid'], None)
eq_(wp['description']['en-US'],
u'Exciting Open Web development action!')
# UTF-8 byte string decoded to unicode.
eq_(wp['description']['es'],
u'\xa1Acci\xf3n abierta emocionante del desarrollo del Web!')
eq_(wp['description']['it'],
u'Azione aperta emozionante di sviluppo di fotoricettore!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def test_no_manifest_at_root(self):
path = self.packaged_app_path('no-manifest-at-root.zip')
if storage_is_remote():
copy_stored_file(path,
path,
src_storage=local_storage,
dst_storage=private_storage)
with self.assertRaises(forms.ValidationError) as exc:
WebAppParser().parse(private_storage.open(path))
m = exc.exception.messages[0]
assert m.startswith('The file "manifest.webapp" was not found'), (
'Unexpected: %s' % m)
def sign(version_id, reviewer=False, resign=False, **kw):
version = Version.objects.get(pk=version_id)
app = version.addon
log.info('Signing version: %s of app: %s' % (version_id, app))
if not app.is_packaged:
log.error('[Webapp:%s] Attempt to sign a non-packaged app.' % app.id)
raise SigningError('Not packaged')
try:
file_obj = version.all_files[0]
except IndexError:
log.error(
'[Webapp:%s] Attempt to sign an app with no files in version.' %
app.id)
raise SigningError('No file')
path = (file_obj.signed_reviewer_file_path if reviewer else
file_obj.signed_file_path)
storage = private_storage if reviewer else public_storage
if storage.exists(path) and not resign:
log.info('[Webapp:%s] Already signed app exists.' % app.id)
return path
if reviewer:
# Reviewers get a unique 'id' so the reviewer installed app won't
# conflict with the public app, and also so multiple versions of the
# same app won't conflict with themselves.
ids = json.dumps({
'id': 'reviewer-{guid}-{version_id}'.format(guid=app.guid,
version_id=version_id),
'version': version_id
})
else:
ids = json.dumps({
'id': app.guid,
'version': version_id
})
with statsd.timer('services.sign.app'):
try:
# Signing starts with the original packaged app file which is
# always on private storage.
sign_app(private_storage.open(file_obj.file_path), path, ids,
reviewer)
except SigningError:
log.info('[Webapp:%s] Signing failed' % app.id)
if storage.exists(path):
storage.delete(path)
raise
log.info('[Webapp:%s] Signing complete.' % app.id)
return path
def compress_export(tarball_name, date):
# We need a temporary directory on the local filesystem that will contain
# all files in order to call `tar`.
local_source_dir = tempfile.mkdtemp()
apps_dirpath = os.path.join(settings.DUMPED_APPS_PATH, 'apps')
# In case apps_dirpath is empty, add a dummy file to make the apps
# directory in the tar archive non-empty. It should not happen in prod, but
# it's nice to have it to prevent the task from failing entirely.
with private_storage.open(
os.path.join(apps_dirpath, '0', '.keep'), 'w') as fd:
fd.write('.')
# Now, copy content from private_storage to that temp directory. We don't
# need to worry about creating the directories locally, the storage class
# does that for us.
for dirpath, dirnames, filenames in walk_storage(
apps_dirpath, storage=private_storage):
for filename in filenames:
src_path = os.path.join(dirpath, filename)
dst_path = os.path.join(
local_source_dir, 'apps', os.path.basename(dirpath), filename)
copy_stored_file(
src_path, dst_path, src_storage=private_storage,
dst_storage=local_storage)
# Also add extra files to the temp directory.
extra_filenames = compile_extra_files(local_source_dir, date)
# All our files are now present locally, let's generate a local filename
# that will contain the final '.tar.gz' before it's copied over to
# public storage.
local_target_file = tempfile.NamedTemporaryFile(
suffix='.tgz', prefix='dumped-apps-')
# tar ALL the things!
cmd = ['tar', 'czf', local_target_file.name, '-C',
local_source_dir] + ['apps'] + extra_filenames
task_log.info(u'Creating dump {0}'.format(local_target_file.name))
subprocess.call(cmd)
# Now copy the local tgz to the public storage.
remote_target_filename = os.path.join(
settings.DUMPED_APPS_PATH, 'tarballs', '%s.tgz' % tarball_name)
copy_stored_file(local_target_file.name, remote_target_filename,
src_storage=local_storage,
dst_storage=public_storage)
# Clean-up.
local_target_file.close()
rm_directory(local_source_dir)
return remote_target_filename
def test_parse(self):
wp = WebAppParser().parse(private_storage.open(self.webapp_path))
eq_(wp['guid'], None)
eq_(wp['description']['en-US'],
u'Exciting Open Web development action!')
# UTF-8 byte string decoded to unicode.
eq_(wp['description']['es'],
u'\xa1Acci\xf3n abierta emocionante del desarrollo del Web!')
eq_(wp['description']['it'],
u'Azione aperta emozionante di sviluppo di fotoricettore!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def test_preview_size(self):
name = 'non-animated.gif'
form = forms.PreviewForm({'upload_hash': name, 'position': 1})
with private_storage.open(os.path.join(self.dest, name), 'wb') as f:
copyfileobj(open(get_image_path(name)), f)
assert form.is_valid(), form.errors
form.save(self.addon)
# Since the task is a post-request-task and we are outside the normal
# request-response cycle, manually send the tasks.
post_request_task._send_tasks()
eq_(self.addon.previews.all()[0].sizes,
{u'image': [250, 297], u'thumbnail': [100, 119]})
def test_packaged_icon(self, save, zip, json):
response = mock.Mock()
response.read.return_value = ''
zf = mock.Mock()
zip.return_value = zf
webapp = app_factory(is_packaged=True)
file_obj = webapp.latest_version.all_files[0]
url = '/path/to/icon.png'
json.return_value = {'icons': {'128': url}}
with private_storage.open(file_obj.file_path, 'w') as f:
f.write("fake zip file")
tasks.fetch_icon(webapp.pk, file_obj.pk)
assert url[1:] in zf.extract_path.call_args[0][0]
def setUp(self):
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True)
# Create a fake package to go along with the app.
self.latest_file = self.app.get_latest_file()
with private_storage.open(self.latest_file.file_path,
mode='w') as package:
test_package = zipfile.ZipFile(package, 'w')
test_package.writestr('manifest.webapp', 'foobar')
test_package.close()
self.latest_file.update(hash=self.latest_file.generate_hash())
self.url = self.app.get_manifest_url()
def get_image(self, filename):
"""Copy image to tmp and return tmp path.
We do this because the task `resize_preview` removes the src file when
finished.
"""
src = get_image_path(filename)
dst = os.path.join(settings.TMP_PATH, 'preview', filename)
with open(src) as local_f:
with private_storage.open(dst, 'w') as remote_f:
shutil.copyfileobj(local_f, remote_f)
return dst
def test_parse_packaged_BOM(self):
path = self.packaged_app_path('mozBOM.zip')
if storage_is_remote():
copy_stored_file(path, path, src_storage=local_storage,
dst_storage=private_storage)
wp = WebAppParser().parse(private_storage.open(path))
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozBOM ょ')
eq_(wp['description']['en-US'], u'Exciting BOM action!')
eq_(wp['description']['es'], u'¡Acción BOM!')
eq_(wp['description']['it'], u'Azione BOM!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def get_image(self, filename):
"""Copy image to tmp and return tmp path.
We do this because the task `resize_preview` removes the src file when
finished.
"""
src = get_image_path(filename)
dst = os.path.join(settings.TMP_PATH, 'preview', filename)
with open(src) as local_f:
with private_storage.open(dst, 'w') as remote_f:
shutil.copyfileobj(local_f, remote_f)
return dst
def test_packaged_icon(self, save, zip, json):
response = mock.Mock()
response.read.return_value = ''
zf = mock.Mock()
zip.return_value = zf
webapp = app_factory(is_packaged=True)
file_obj = webapp.latest_version.all_files[0]
url = '/path/to/icon.png'
json.return_value = {'icons': {'128': url}}
with private_storage.open(file_obj.file_path, 'w') as f:
f.write("fake zip file")
tasks.fetch_icon(webapp.pk, file_obj.pk)
assert url[1:] in zf.extract_path.call_args[0][0]
def setUp(self):
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True)
# Create a fake package to go along with the app.
self.latest_file = self.app.get_latest_file()
with private_storage.open(self.latest_file.file_path,
mode='w') as package:
test_package = zipfile.ZipFile(package, 'w')
test_package.writestr('manifest.webapp', 'foobar')
test_package.close()
self.latest_file.update(hash=self.latest_file.generate_hash())
self.url = self.app.get_manifest_url()
def test_delete_with_file(self):
"""Test that when a Extension instance is deleted, the corresponding
file on the filesystem is also deleted."""
extension = Extension.objects.create(version='0.1')
file_path = extension.file_path
with private_storage.open(file_path, 'w') as f:
f.write('sample data\n')
assert private_storage.exists(file_path)
try:
extension.delete()
assert not private_storage.exists(file_path)
finally:
if private_storage.exists(file_path):
private_storage.delete(file_path)
def test_parse_packaged_BOM(self):
path = self.packaged_app_path('mozBOM.zip')
if storage_is_remote():
copy_stored_file(path,
path,
src_storage=local_storage,
dst_storage=private_storage)
wp = WebAppParser().parse(private_storage.open(path))
eq_(wp['guid'], None)
eq_(wp['name']['en-US'], u'Packaged MozBOM ょ')
eq_(wp['description']['en-US'], u'Exciting BOM action!')
eq_(wp['description']['es'], u'¡Acción BOM!')
eq_(wp['description']['it'], u'Azione BOM!')
eq_(wp['version'], '1.0')
eq_(wp['default_locale'], 'en-US')
def save_promo_imgs(obj, img_content):
"""
Saves the promo image for `obj` to its final destination.
`obj` can be an app or a website.
"""
tmp_dst = os.path.join(settings.TMP_PATH, 'promo_imgs', uuid.uuid4().hex)
with private_storage.open(tmp_dst, 'wb') as fd:
fd.write(img_content)
dirname = obj.get_promo_img_dir()
destination = os.path.join(dirname, '%s' % obj.pk)
remove_promo_imgs(destination)
resize_promo_imgs(
tmp_dst, destination, mkt.PROMO_IMG_SIZES,
set_modified_on=[obj])
def setUp(self):
self.app = Webapp.objects.get(pk=337141)
self.app.update(is_packaged=True)
# Create a fake package to go along with the app.
self.latest_file = self.app.get_latest_file()
with private_storage.open(self.latest_file.file_path,
mode='w') as package:
test_package = zipfile.ZipFile(package, 'w')
test_package.writestr('manifest.webapp', 'foobar')
test_package.close()
self.latest_file.update(hash=self.latest_file.generate_hash())
self.url = self.app.get_manifest_url()
# Don't count things left over from setup, so assertNumQueries will be
# accurate.
_send_tasks()
def test_strip_utf8_bom(self):
with self.patch_requests() as ur:
with open(self.file('utf8bom.webapp')) as fp:
content = fp.read()
ur.iter_content.return_value = mock.Mock(next=lambda: content)
tasks.fetch_manifest('url', self.upload.pk)
# Should not be called with anything else (e.g., `decode_unicode`).
ur.iter_content.assert_called_with(
chunk_size=settings.MAX_WEBAPP_UPLOAD_SIZE + 1)
upload = self.get_upload()
with private_storage.open(upload.path, 'rb') as fp:
manifest = fp.read()
json.loads(manifest) # No parse error.
assert not manifest.startswith(codecs.BOM_UTF8)