def path(): # Check file paths / permissions read_and_write = ( settings.TMP_PATH, settings.MEDIA_ROOT, user_media_path('addons'), user_media_path('guarded_addons'), user_media_path('addon_icons'), user_media_path('previews'), user_media_path('userpics'),) read_only = [os.path.join(settings.ROOT, 'locale')] filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write') for path in read_and_write] filepaths += [(path, os.R_OK, 'We want read') for path in read_only] filepath_results = [] filepath_status = True for path, perms, notes in filepaths: path_exists = os.path.exists(path) path_perms = os.access(path, perms) filepath_status = filepath_status and path_exists and path_perms if not isinstance(path, bytes): notes += ' / should be a bytestring!' filepath_results.append((path, path_exists, path_perms, notes)) status = filepath_status status = '' if not filepath_status: status = 'check main status page for broken perms / values' return status, filepath_results
def migrate_legacy_dictionary_to_webextension(addon): """Migrate a single legacy dictionary to webextension format, creating a new package from the current_version, faking an upload to create a new Version instance.""" user = UserProfile.objects.get(pk=settings.TASK_USER_ID) now = datetime.now() # Wrap zip in FileUpload for Version.from_upload() to consume. upload = FileUpload.objects.create(user=user, valid=True) destination = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_webext_dictionary_from_legacy(addon, destination) upload.update(path=destination) parsed_data = parse_addon(upload, user=user) # Create version. version = Version.from_upload(upload, addon, platforms=[amo.PLATFORM_ALL.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user) # Sign the file, and set it to public. That should automatically set # current_version to the version we created. file_ = version.all_files[0] sign_file(file_) file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def migrate_legacy_dictionary_to_webextension(addon): """Migrate a single legacy dictionary to webextension format, creating a new package from the current_version, faking an upload to create a new Version instance.""" user = UserProfile.objects.get(pk=settings.TASK_USER_ID) now = datetime.now() # Wrap zip in FileUpload for Version.from_upload() to consume. upload = FileUpload.objects.create( user=user, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') target_language = build_webext_dictionary_from_legacy(addon, destination) if not addon.target_locale: addon.update(target_locale=target_language) upload.update(path=destination) parsed_data = parse_addon(upload, addon=addon, user=user) # Create version. # WebExtension dictionaries are only compatible with Firefox Desktop # Firefox for Android uses the OS spellchecking. version = Version.from_upload( upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user) # Sign the file, and set it to public. That should automatically set # current_version to the version we created. file_ = version.all_files[0] sign_file(file_) file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def new_theme_version_with_69_properties(old_version): timer = StopWatch( 'addons.tasks.repack_themes_for_69.new_theme_version.') timer.start() author = get_user() # Wrap zip in FileUpload for Version from_upload to consume. upload = FileUpload.objects.create(user=author, valid=True) filename = uuid.uuid4().hex + '.xpi' destination = os.path.join(user_media_path('addons'), 'temp', filename) old_xpi = get_filepath(old_version.all_files[0]) build_69_compatible_theme( old_xpi, destination, get_next_version_number(old_version.addon)) upload.update(path=destination, name=filename) timer.log_interval('1.build_xpi') # Create addon + version parsed_data = parse_addon(upload, addon=old_version.addon, user=author) timer.log_interval('2.parse_addon') version = Version.from_upload( upload, old_version.addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) timer.log_interval('3.initialize_version') # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update( reviewed=datetime.now(), status=amo.STATUS_APPROVED) timer.log_interval('4.sign_files') return version
def save_theme_reupload(header, footer, addon, **kw): header_dst = None footer_dst = None dst_root = os.path.join(user_media_path('addons'), str(addon.id)) try: if header: header = os.path.join(settings.TMP_PATH, 'persona_header', header) header_dst = os.path.join(dst_root, 'pending_header.png') save_persona_image(src=header, full_dst=header_dst) if footer: footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer) footer_dst = os.path.join(dst_root, 'pending_footer.png') save_persona_image(src=footer, full_dst=footer_dst) except IOError as e: log.error(str(e)) raise if header_dst or footer_dst: theme = addon.persona header = 'pending_header.png' if header_dst else theme.header # Theme footer is optional, but can't be None. footer = theme.footer or '' if footer_dst: footer = 'pending_footer.png' # Store pending header and/or footer file paths for review. RereviewQueueTheme.objects.filter(theme=theme).delete() rqt = RereviewQueueTheme(theme=theme, header=header, footer=footer) rereviewqueuetheme_checksum(rqt=rqt) rqt.save()
def rezip_file(response, pk): # An .xpi does not have a directory inside the zip, yet zips from github # do, so we'll need to rezip the file before passing it through to the # validator. loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) old_filename = '{}_github_webhook.zip'.format(pk) old_path = os.path.join(loc, old_filename) with storage.open(old_path, 'wb') as old: old.write(response.content) new_filename = '{}_github_webhook.xpi'.format(pk) new_path = os.path.join(loc, new_filename) old_zip = SafeUnzip(old_path) if not old_zip.is_valid(): raise with storage.open(new_path, 'w') as new: new_zip = zipfile.ZipFile(new, 'w') for obj in old_zip.filelist: # Basically strip off the leading directory. new_filename = obj.filename.partition('/')[-1] if not new_filename: continue new_zip.writestr(new_filename, old_zip.read(obj.filename)) new_zip.close() old_zip.close() return new_path
def test_reupload(self, save_persona_image_mock, create_persona_preview_images_mock, make_checksum_mock): make_checksum_mock.return_value = 'checksumbeforeyouwrecksome' data = self.get_dict(header_hash='y0l0', footer_hash='abab') self.form = EditThemeForm(data, request=self.request, instance=self.instance) assert self.form.is_valid() self.form.save() dst = os.path.join(user_media_path('addons'), str(self.instance.id)) header_src = os.path.join(settings.TMP_PATH, 'persona_header', u'y0l0') footer_src = os.path.join(settings.TMP_PATH, 'persona_footer', u'abab') assert save_persona_image_mock.mock_calls == ( [mock.call(src=header_src, full_dst=os.path.join(dst, 'pending_header.png')), mock.call(src=footer_src, full_dst=os.path.join(dst, 'pending_footer.png'))]) rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona) assert rqt.count() == 1 assert rqt[0].header == 'pending_header.png' assert rqt[0].footer == 'pending_footer.png' assert not rqt[0].dupe_persona
def migrate_legacy_dictionary_to_webextension(addon): """Migrate a single legacy dictionary to webextension format, creating a new package from the current_version, faking an upload to create a new Version instance.""" user = UserProfile.objects.get(pk=settings.TASK_USER_ID) now = datetime.now() # Wrap zip in FileUpload for Version.from_upload() to consume. upload = FileUpload.objects.create(user=user, valid=True) destination = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') target_language = build_webext_dictionary_from_legacy(addon, destination) if not addon.target_locale: addon.update(target_locale=target_language) upload.update(path=destination) parsed_data = parse_addon(upload, addon=addon, user=user) # Create version. # WebExtension dictionaries are only compatible with Firefox Desktop # Firefox for Android uses the OS spellchecking. version = Version.from_upload(upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user) # Sign the file, and set it to public. That should automatically set # current_version to the version we created. file_ = version.all_files[0] sign_file(file_) file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def test_reupload(self, save_persona_image_mock, create_persona_preview_images_mock, make_checksum_mock): make_checksum_mock.return_value = 'checksumbeforeyouwrecksome' data = self.get_dict(header_hash='y0l0', footer_hash='abab') self.form = EditThemeForm(data, request=self.request, instance=self.instance) assert self.form.is_valid() self.form.save() dst = os.path.join(user_media_path('addons'), str(self.instance.id)) header_src = os.path.join(settings.TMP_PATH, 'persona_header', u'y0l0') footer_src = os.path.join(settings.TMP_PATH, 'persona_footer', u'abab') assert save_persona_image_mock.mock_calls == ([ mock.call(src=header_src, full_dst=os.path.join(dst, 'pending_header.png')), mock.call(src=footer_src, full_dst=os.path.join(dst, 'pending_footer.png')) ]) rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona) assert rqt.count() == 1 assert rqt[0].header == 'pending_header.png' assert rqt[0].footer == 'pending_footer.png' assert not rqt[0].dupe_persona
def rezip_file(response, pk): # An .xpi does not have a directory inside the zip, yet zips from github # do, so we'll need to rezip the file before passing it through to the # validator. loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) old_filename = '{}_github_webhook.zip'.format(pk) old_path = os.path.join(loc, old_filename) with storage.open(old_path, 'wb') as old: old.write(response.content) new_filename = '{}_github_webhook.xpi'.format(pk) new_path = os.path.join(loc, new_filename) old_zip = SafeZip(old_path) if not old_zip.is_valid: raise with storage.open(new_path, 'w') as new: new_zip = zipfile.ZipFile(new, 'w') for obj in old_zip.filelist: # Basically strip off the leading directory. new_filename = obj.filename.partition('/')[-1] if not new_filename: continue new_zip.writestr(new_filename, old_zip.read(obj.filename)) new_zip.close() old_zip.close() return new_path
def test_success(self, save_persona_image_mock, create_persona_preview_images_mock, make_checksum_mock): make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself' self.request.user = UserProfile.objects.get(pk=2519) data = self.get_dict() header_url = self.get_img_urls() # Upload header image. img = open(get_image_path('persona-header.jpg'), 'rb') r_ajax = self.client.post(header_url, {'upload_image': img}) content = json.loads(force_text(r_ajax.content)) data.update(header_hash=content['upload_hash']) # Populate and save form. self.post() assert self.form.is_valid(), self.form.errors self.form.save() addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0] persona = addon.persona # Test for correct Addon and Persona values. assert six.text_type(addon.name) == data['name'] assert addon.slug == data['slug'] self.assertSetEqual(set(addon.categories.values_list('id', flat=True)), {self.cat.id}) self.assertSetEqual(set(addon.tags.values_list('tag_text', flat=True)), set(data['tags'].split(', '))) assert persona.persona_id == 0 assert persona.license == data['license'] assert persona.accentcolor == data['accentcolor'].lstrip('#') assert persona.textcolor == data['textcolor'].lstrip('#') assert persona.author == self.request.user.username assert persona.display_username == self.request.user.name assert not persona.dupe_persona v = addon.versions.all() assert len(v) == 1 assert v[0].version == '0' # Test for header and preview images. dst = os.path.join(user_media_path('addons'), str(addon.id)) header_src = os.path.join(settings.TMP_PATH, 'persona_header', u'b4ll1n') assert save_persona_image_mock.mock_calls == ([ mock.call(src=header_src, full_dst=os.path.join(dst, 'header.png')) ]) create_persona_preview_images_mock.assert_called_with( src=header_src, full_dst=[ os.path.join(dst, 'preview.png'), os.path.join(dst, 'icon.png') ], set_modified_on=addon.serializable_reference())
def delete_persona_image(dst, **kw): log.info('[1@None] Deleting persona image: %s.' % dst) if not dst.startswith(user_media_path('addons')): log.error("Someone tried deleting something they shouldn't: %s" % dst) return try: storage.delete(dst) except Exception, e: log.error('Error deleting persona image: %s' % e)
def delete_persona_image(dst, **kw): log.info('[1@None] Deleting persona image: %s.' % dst) if not dst.startswith(user_media_path('addons')): log.error("Someone tried deleting something they shouldn't: %s" % dst) return try: storage.delete(dst) except Exception as e: log.error('Error deleting persona image: %s' % e)
def picture_dir(self): from olympia.amo.templatetags.jinja_helpers import user_media_path split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id)) return os.path.join( user_media_path('userpics'), split_id.group(2) or '0', split_id.group(1) or '0', )
def add_static_theme_from_lwt(lwt): # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) # Version.from_upload sorts out platforms for us. version = Version.from_upload( upload, addon, platforms=None, channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get( amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) log.debug('New static theme %r created from %r' % (addon, lwt)) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update( datestatuschanged=datetime.now(), reviewed=datetime.now(), status=amo.STATUS_PUBLIC) addon.update(status=amo.STATUS_PUBLIC) return addon
def test_success(self, save_persona_image_mock, create_persona_preview_images_mock, make_checksum_mock): make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself' self.request.user = UserProfile.objects.get(pk=2519) data = self.get_dict() header_url = self.get_img_urls() # Upload header image. img = open(get_image_path('persona-header.jpg'), 'rb') r_ajax = self.client.post(header_url, {'upload_image': img}) data.update(header_hash=json.loads(r_ajax.content)['upload_hash']) # Populate and save form. self.post() assert self.form.is_valid(), self.form.errors self.form.save() addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0] persona = addon.persona # Test for correct Addon and Persona values. assert unicode(addon.name) == data['name'] assert addon.slug == data['slug'] self.assertSetEqual(set(addon.categories.values_list('id', flat=True)), {self.cat.id}) self.assertSetEqual(set(addon.tags.values_list('tag_text', flat=True)), set(data['tags'].split(', '))) assert persona.persona_id == 0 assert persona.license == data['license'] assert persona.accentcolor == data['accentcolor'].lstrip('#') assert persona.textcolor == data['textcolor'].lstrip('#') assert persona.author == self.request.user.username assert persona.display_username == self.request.user.name assert not persona.dupe_persona v = addon.versions.all() assert len(v) == 1 assert v[0].version == '0' # Test for header and preview images. dst = os.path.join(user_media_path('addons'), str(addon.id)) header_src = os.path.join(settings.TMP_PATH, 'persona_header', u'b4ll1n') assert save_persona_image_mock.mock_calls == ( [mock.call(src=header_src, full_dst=os.path.join(dst, 'header.png'))]) create_persona_preview_images_mock.assert_called_with( src=header_src, full_dst=[os.path.join(dst, 'preview.png'), os.path.join(dst, 'icon.png')], set_modified_on=addon.serializable_reference())
def _image_path(self, folder, file_ext): from olympia.amo.templatetags.jinja_helpers import user_media_path url = os.path.join( user_media_path(self.media_folder), folder, str(self.id // 1000), f'{self.id}.{file_ext}', ) return url
def get_background_image_urls(self): if self.addon.type != amo.ADDON_STATICTHEME: return [] background_images_folder = os.path.join( user_media_path('addons'), str(self.addon.id), unicode(self.id)) background_images_url = '/'.join( (user_media_url('addons'), str(self.addon.id), unicode(self.id))) out = [ background.replace(background_images_folder, background_images_url) for background in walkfiles(background_images_folder)] return out
def delete_icon(dst, **kw): log.info('[1@None] Deleting icon: %s.' % dst) if not dst.startswith(user_media_path('collection_icons')): log.error("Someone tried deleting something they shouldn't: %s" % dst) return try: storage.delete(dst) except Exception, e: log.error("Error deleting icon: %s" % e)
def delete_photo(dst, **kw): task_log.debug('[1@None] Deleting photo: %s.' % dst) if not dst.startswith(user_media_path('userpics')): task_log.error("Someone tried deleting something they shouldn't: %s" % dst) return try: storage.delete(dst) except Exception, e: task_log.error("Error deleting userpic: %s" % e)
def delete_photo(dst, **kw): task_log.info('[1@None] Deleting photo: %s.' % dst) if not dst.startswith(user_media_path('userpics')): task_log.error("Someone tried deleting something they shouldn't: %s" % dst) return try: storage.delete(dst) except Exception as e: task_log.error('Error deleting userpic: %s' % e)
def path(): # Check file paths / permissions rw = ( settings.TMP_PATH, settings.MEDIA_ROOT, user_media_path('addons'), user_media_path('guarded_addons'), user_media_path('addon_icons'), user_media_path('previews'), user_media_path('userpics'), dump_apps.Command.get_json_path(), ) r = [ os.path.join(settings.ROOT, 'locale'), # The deploy process will want write access to this. # We do not want Django to have write access though. settings.PROD_DETAILS_DIR ] filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write') for path in rw] filepaths += [(path, os.R_OK, 'We want read') for path in r] filepath_results = [] filepath_status = True for path, perms, notes in filepaths: path_exists = os.path.exists(path) path_perms = os.access(path, perms) filepath_status = filepath_status and path_exists and path_perms if not isinstance(path, str): notes += ' / should be a bytestring!' filepath_results.append((path, path_exists, path_perms, notes)) status = filepath_status status = '' if not filepath_status: status = 'check main status page for broken perms / values' return status, filepath_results
def add_static_theme_from_lwt(lwt): # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) # Version.from_upload sorts out platforms for us. version = Version.from_upload( upload, addon, platforms=None, channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get( amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) log.debug('New static theme %r created from %r' % (addon, lwt)) # And finally update the statuses version.all_files[0].update(status=amo.STATUS_PUBLIC) addon.update(status=amo.STATUS_PUBLIC) return addon
def save_theme(header, addon, **kw): """Save theme image and calculates checksum after theme save.""" dst_root = os.path.join(user_media_path('addons'), str(addon.id)) header = os.path.join(settings.TMP_PATH, 'persona_header', header) header_dst = os.path.join(dst_root, 'header.png') try: save_persona_image(src=header, full_dst=header_dst) create_persona_preview_images( src=header, full_dst=[os.path.join(dst_root, 'preview.png'), os.path.join(dst_root, 'icon.png')], set_modified_on=[addon]) theme_checksum(addon.persona) except IOError: addon.delete() raise
def save_theme_reupload(header, addon_pk, **kw): addon = Addon.objects.get(pk=addon_pk) header_dst = None dst_root = os.path.join(user_media_path('addons'), str(addon.id)) try: if header: header = os.path.join(settings.TMP_PATH, 'persona_header', header) header_dst = os.path.join(dst_root, 'pending_header.png') save_persona_image(src=header, full_dst=header_dst) except IOError as e: log.error(str(e)) raise if header_dst: theme = addon.persona header = 'pending_header.png' if header_dst else theme.header
def save_theme(header, addon_pk, **kw): """Save theme image and calculates checksum after theme save.""" addon = Addon.objects.get(pk=addon_pk) dst_root = os.path.join(user_media_path('addons'), str(addon.id)) header = os.path.join(settings.TMP_PATH, 'persona_header', header) header_dst = os.path.join(dst_root, 'header.png') try: save_persona_image(src=header, full_dst=header_dst) create_persona_preview_images( src=header, full_dst=[os.path.join(dst_root, 'preview.png'), os.path.join(dst_root, 'icon.png')], set_modified_on=addon.serializable_reference()) theme_checksum(addon.persona) except IOError: addon.delete() raise
def _uploader(resize_size, final_size): img = get_image_path('mozilla.png') original_size = (339, 128) src = tempfile.NamedTemporaryFile(mode='r+b', suffix='.png', delete=False, dir=settings.TMP_PATH) if not isinstance(final_size, list): final_size = [final_size] resize_size = [resize_size] uploadto = user_media_path('addon_icons') try: os.makedirs(uploadto) except OSError: pass for rsize, expected_size in zip(resize_size, final_size): # resize_icon moves the original shutil.copyfile(img, src.name) src_image = Image.open(src.name) assert src_image.size == original_size dest_name = os.path.join(uploadto, '1234') with mock.patch('olympia.amo.utils.pngcrush_image') as pngcrush_mock: return_value = tasks.resize_icon(src.name, dest_name, [rsize]) dest_image = '%s-%s.png' % (dest_name, rsize) assert pngcrush_mock.call_count == 1 assert pngcrush_mock.call_args_list[0][0][0] == dest_image assert image_size(dest_image) == expected_size # original should have been moved to -original orig_image = '%s-original.png' % dest_name assert os.path.exists(orig_image) # Return value of the task should be a dict with an icon_hash key # containing the 8 first chars of the md5 hash of the source file, # which is bb362450b00f0461c6bddc6b97b3c30b. assert return_value == {'icon_hash': 'bb362450'} os.remove(dest_image) assert not os.path.exists(dest_image) os.remove(orig_image) assert not os.path.exists(orig_image) shutil.rmtree(uploadto) assert not os.path.exists(src.name)
def add_file(self, chunks, filename, size): if not self.uuid: self.uuid = self._meta.get_field('uuid')._create_uuid() filename = force_text(u'{0}_{1}'.format(self.uuid.hex, filename)) loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(filename) is_crx = False # Change a ZIP to an XPI, to maintain backward compatibility # with older versions of Firefox and to keep the rest of the XPI code # path as consistent as possible for ZIP uploads. # See: https://github.com/mozilla/addons-server/pull/2785 if ext == '.zip': ext = '.xpi' # If the extension is a CRX, we need to do some actual work to it # before we just convert it to an XPI. We strip the header from the # CRX, then it's good; see more about the CRX file format here: # https://developer.chrome.com/extensions/crx if ext == '.crx': ext = '.xpi' is_crx = True if ext in amo.VALID_ADDON_FILE_EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc), extra={ 'email': (self.user.email if self.user and self.user.email else '') }) if is_crx: hash_func = write_crx_as_xpi(chunks, loc) else: hash_func = hashlib.sha256() with storage.open(loc, 'wb') as file_destination: for chunk in chunks: hash_func.update(chunk) file_destination.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash_func.hexdigest() self.save()
def _uploader(resize_size, final_size): img = get_image_path('mozilla.png') original_size = (339, 128) src = tempfile.NamedTemporaryFile( mode='r+w+b', suffix='.png', delete=False, dir=settings.TMP_PATH) if not isinstance(final_size, list): final_size = [final_size] resize_size = [resize_size] uploadto = user_media_path('addon_icons') try: os.makedirs(uploadto) except OSError: pass for rsize, expected_size in zip(resize_size, final_size): # resize_icon moves the original shutil.copyfile(img, src.name) src_image = Image.open(src.name) assert src_image.size == original_size dest_name = os.path.join(uploadto, '1234') with mock.patch('olympia.amo.utils.pngcrush_image') as pngcrush_mock: return_value = tasks.resize_icon(src.name, dest_name, [rsize]) dest_image = '%s-%s.png' % (dest_name, rsize) assert pngcrush_mock.call_count == 1 assert pngcrush_mock.call_args_list[0][0][0] == dest_image assert image_size(dest_image) == expected_size # original should have been moved to -original orig_image = '%s-original.png' % dest_name assert os.path.exists(orig_image) # Return value of the task should be a dict with an icon_hash key # containing the 8 first chars of the md5 hash of the source file, # which is bb362450b00f0461c6bddc6b97b3c30b. assert return_value == {'icon_hash': 'bb362450'} os.remove(dest_image) assert not os.path.exists(dest_image) os.remove(orig_image) assert not os.path.exists(orig_image) shutil.rmtree(uploadto) assert not os.path.exists(src.name)
def create_theme_images(theme, placement, hash_): """ Generates 2 images, one in the temp folder and the other in the user-media one. Both are needed to generate previews for themes. """ color = random.choice(ImageColor.colormap.keys()) image = Image.new('RGB', (3000, 200), color) tmp_path = os.path.join(settings.TMP_PATH, 'persona_{placement}'.format(placement=placement)) if not os.path.exists(tmp_path): os.makedirs(tmp_path) tmp_loc = os.path.join(tmp_path, hash_) image.save(tmp_loc, 'jpeg') media_path = os.path.join(user_media_path('addons'), str(theme.id)) if not os.path.exists(media_path): os.makedirs(media_path) media_loc = os.path.join(media_path, hash_) image.save(media_loc, 'jpeg')
def add_file(self, chunks, filename, size): if not self.uuid: self.uuid = self._meta.get_field('uuid')._create_uuid() _base, ext = os.path.splitext(filename) was_crx = ext == '.crx' # Filename we'll expose (but not use for storage). self.name = force_str('{0}_{1}'.format(self.uuid.hex, filename)) # Final path on our filesystem. If it had a valid extension we change # it to .xpi (CRX files are converted before validation, so they will # be treated as normal .xpi for validation). If somehow this is # not a valid archive or the extension is invalid parse_addon() will # eventually complain at validation time or before even reaching the # linter. if ext in amo.VALID_ADDON_FILE_EXTENSIONS: ext = '.xpi' self.path = os.path.join(user_media_path('addons'), 'temp', '{0}{1}'.format(uuid.uuid4().hex, ext)) hash_obj = None if was_crx: try: hash_obj = write_crx_as_xpi(chunks, self.path) except InvalidOrUnsupportedCrx: # We couldn't convert the crx file. Write it to the filesystem # normally, the validation process should reject this with a # proper error later. pass if hash_obj is None: hash_obj = self.write_data_to_path(chunks) self.hash = 'sha256:%s' % hash_obj.hexdigest() # The following log statement is used by foxsec-pipeline. log.info( 'UPLOAD: %r (%s bytes) to %r' % (self.name, size, self.path), extra={ 'email': (self.user.email if self.user and self.user.email else ''), 'upload_hash': self.hash, }, ) self.save()
def save_theme(header, footer, addon, **kw): """Save theme image and calculates checksum after theme save.""" dst_root = os.path.join(user_media_path('addons'), str(addon.id)) header = os.path.join(settings.TMP_PATH, 'persona_header', header) header_dst = os.path.join(dst_root, 'header.png') if footer: footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer) footer_dst = os.path.join(dst_root, 'footer.png') try: save_persona_image(src=header, full_dst=header_dst) if footer: save_persona_image(src=footer, full_dst=footer_dst) create_persona_preview_images( src=header, full_dst=[os.path.join(dst_root, 'preview.png'), os.path.join(dst_root, 'icon.png')], set_modified_on=[addon]) theme_checksum(addon.persona) except IOError: addon.delete() raise
def approve_rereview(theme): """Replace original theme with pending theme on filesystem.""" # If reuploaded theme, replace old theme design. storage = LocalFileStorage() rereview = theme.rereviewqueuetheme_set.all() reupload = rereview[0] if reupload.header_path != reupload.theme.header_path: create_persona_preview_images( src=reupload.header_path, full_dst=[ reupload.theme.thumb_path, reupload.theme.icon_path], set_modified_on=[reupload.theme.addon]) if not reupload.theme.is_new(): # Legacy themes also need a preview_large.jpg. # Modern themes use preview.png for both thumb and preview so there # is no problem there. copy_stored_file(reupload.theme.thumb_path, reupload.theme.preview_path, storage=storage) move_stored_file( reupload.header_path, reupload.theme.header_path, storage=storage) theme = reupload.theme footer_path = theme.footer_path if reupload.footer_path != footer_path: if not footer_path: dst_root = os.path.join(user_media_path('addons'), str(theme.addon.id)) footer_path = os.path.join(dst_root, 'footer.png') theme.footer = 'footer.png' theme.save() move_stored_file( reupload.footer_path, footer_path, storage=storage) rereview.delete() theme.addon.increment_theme_version_number()
def add_file(self, chunks, filename, size): if not self.uuid: self.uuid = self._meta.get_field('uuid')._create_uuid() filename = force_bytes(u'{0}_{1}'.format(self.uuid.hex, filename)) loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) base, ext = os.path.splitext(force_bytes(filename)) is_crx = False # Change a ZIP to an XPI, to maintain backward compatibility # with older versions of Firefox and to keep the rest of the XPI code # path as consistent as possible for ZIP uploads. # See: https://github.com/mozilla/addons-server/pull/2785 if ext == '.zip': ext = '.xpi' # If the extension is a CRX, we need to do some actual work to it # before we just convert it to an XPI. We strip the header from the # CRX, then it's good; see more about the CRX file format here: # https://developer.chrome.com/extensions/crx if ext == '.crx': ext = '.xpi' is_crx = True if ext in EXTENSIONS: loc += ext log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc)) if is_crx: hash = write_crx_as_xpi(chunks, storage, loc) else: hash = hashlib.sha256() with storage.open(loc, 'wb') as file_destination: for chunk in chunks: hash.update(chunk) file_destination.write(chunk) self.path = loc self.name = filename self.hash = 'sha256:%s' % hash.hexdigest() self.save()
def _uploader(resize_size, final_size): img = get_image_path('mozilla.png') original_size = (339, 128) src = tempfile.NamedTemporaryFile(mode='r+w+b', suffix=".png", delete=False) # resize_icon removes the original shutil.copyfile(img, src.name) src_image = Image.open(src.name) assert src_image.size == original_size if isinstance(final_size, list): uploadto = user_media_path('addon_icons') try: os.makedirs(uploadto) except OSError: pass for rsize, fsize in zip(resize_size, final_size): dest_name = os.path.join(uploadto, '1234') tasks.resize_icon(src.name, dest_name, resize_size, locally=True) dest_image = Image.open(open('%s-%s.png' % (dest_name, rsize))) assert dest_image.size == fsize if os.path.exists(dest_image.filename): os.remove(dest_image.filename) assert not os.path.exists(dest_image.filename) shutil.rmtree(uploadto) else: dest = tempfile.mktemp(suffix='.png') tasks.resize_icon(src.name, dest, resize_size, locally=True) dest_image = Image.open(dest) assert dest_image.size == final_size assert not os.path.exists(src.name)
def save_theme_reupload(header, addon_pk, **kw): addon = Addon.objects.get(pk=addon_pk) header_dst = None dst_root = os.path.join(user_media_path('addons'), str(addon.id)) try: if header: header = os.path.join(settings.TMP_PATH, 'persona_header', header) header_dst = os.path.join(dst_root, 'pending_header.png') save_persona_image(src=header, full_dst=header_dst) except IOError as e: log.error(str(e)) raise if header_dst: theme = addon.persona header = 'pending_header.png' if header_dst else theme.header # Store pending header file paths for review. RereviewQueueTheme.objects.filter(theme=theme).delete() rqt = RereviewQueueTheme(theme=theme, header=header) rereviewqueuetheme_checksum(rqt=rqt) rqt.save()
def path(): # Check file paths / permissions rw = (settings.TMP_PATH, settings.MEDIA_ROOT, user_media_path('addons'), user_media_path('guarded_addons'), user_media_path('addon_icons'), user_media_path('collection_icons'), user_media_path('previews'), user_media_path('userpics'), user_media_path('reviewer_attachments'), dump_apps.Command.get_json_path(),) r = [os.path.join(settings.ROOT, 'locale'), # The deploy process will want write access to this. # We do not want Django to have write access though. settings.PROD_DETAILS_DIR] filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write') for path in rw] filepaths += [(path, os.R_OK, 'We want read') for path in r] filepath_results = [] filepath_status = True for path, perms, notes in filepaths: path_exists = os.path.exists(path) path_perms = os.access(path, perms) filepath_status = filepath_status and path_exists and path_perms if not isinstance(path, str): notes += ' / should be a bytestring!' filepath_results.append((path, path_exists, path_perms, notes)) status = filepath_status status = '' if not filepath_status: status = 'check main status page for broken perms / values' return status, filepath_results
def test_with_settings(self): path = jinja_helpers.user_media_path('addons') assert path == '/another/path/'
def test_without_settings(self): del settings.ADDONS_PATH path = jinja_helpers.user_media_path('addons') assert path == '/path/addons'
def guarded_file_path(self): return os.path.join(user_media_path('guarded_addons'), str(self.version.addon_id), self.filename)
def gc(test_result=True): """Site-wide garbage collections.""" def days_ago(days): return datetime.today() - timedelta(days=days) log.debug('Collecting data to delete') logs = (ActivityLog.objects.filter(created__lt=days_ago(90)).exclude( action__in=amo.LOG_KEEP).values_list('id', flat=True)) # Paypal only keeps retrying to verify transactions for up to 3 days. If we # still have an unverified transaction after 6 days, we might as well get # rid of it. contributions_to_delete = (Contribution.objects.filter( transaction_id__isnull=True, created__lt=days_ago(6)).values_list('id', flat=True)) collections_to_delete = (Collection.objects.filter( created__lt=days_ago(2), type=amo.COLLECTION_ANONYMOUS).values_list('id', flat=True)) for chunk in chunked(logs, 100): tasks.delete_logs.delay(chunk) for chunk in chunked(contributions_to_delete, 100): tasks.delete_stale_contributions.delay(chunk) for chunk in chunked(collections_to_delete, 100): tasks.delete_anonymous_collections.delay(chunk) # Incomplete addons cannot be deleted here because when an addon is # rejected during a review it is marked as incomplete. See bug 670295. log.debug('Cleaning up test results extraction cache.') # lol at check for '/' if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/': cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name', 'validate-*', '-mtime', '+7', '-type', 'd', '-exec', 'rm', '-rf', "{}", ';') output = Popen(cmd, stdout=PIPE).communicate()[0] for line in output.split("\n"): log.debug(line) else: log.warning('MEDIA_ROOT not defined.') if user_media_path('collection_icons'): log.debug('Cleaning up uncompressed icons.') cmd = ('find', user_media_path('collection_icons'), '-name', '*__unconverted', '-mtime', '+1', '-type', 'f', '-exec', 'rm', '{}', ';') output = Popen(cmd, stdout=PIPE).communicate()[0] for line in output.split("\n"): log.debug(line) USERPICS_PATH = user_media_path('userpics') if USERPICS_PATH: log.debug('Cleaning up uncompressed userpics.') cmd = ('find', USERPICS_PATH, '-name', '*__unconverted', '-mtime', '+1', '-type', 'f', '-exec', 'rm', '{}', ';') output = Popen(cmd, stdout=PIPE).communicate()[0] for line in output.split("\n"): log.debug(line) # Delete stale FileUploads. stale_uploads = FileUpload.objects.filter( created__lte=days_ago(180)).order_by('id') for file_upload in stale_uploads: log.debug(u'[FileUpload:{uuid}] Removing file: {path}'.format( uuid=file_upload.uuid, path=file_upload.path)) if file_upload.path: try: storage.delete(file_upload.path) except OSError: pass file_upload.delete()
def _image_path(self, url_template): from olympia.amo.templatetags.jinja_helpers import user_media_path args = [user_media_path(self.media_folder), self.id // 1000, self.id] return url_template % tuple(args)
def add_static_theme_from_lwt(lwt): from olympia.activity.models import AddonLog olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID)) # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create(user=author, valid=True) destination = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload(parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) addon_updates = {} # static themes are only compatible with Firefox at the moment, # not Android version = Version.from_upload(upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get(amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create(addon=addon, category=Category.from_static_category( static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Steal the ratings (even with soft delete they'll be deleted anyway) addon_updates.update(average_rating=lwt.average_rating, bayesian_rating=lwt.bayesian_rating, total_ratings=lwt.total_ratings, text_ratings_count=lwt.text_ratings_count) Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version) # Modify the activity log entry too. rating_activity_log_ids = [ l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review' ] addonlog_qs = AddonLog.objects.filter( addon=lwt, activity_log__action__in=rating_activity_log_ids) [alog.transfer(addon) for alog in addonlog_qs.iterator()] # Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats # dashboard and future update counts, and copy the summary numbers for now. migrate_theme_update_count(lwt, addon) addon_updates.update(average_daily_users=lwt.persona.popularity or 0, hotness=lwt.persona.movers or 0) # Logging activity.log_create(amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update(datestatuschanged=lwt.last_updated, reviewed=datetime.now(), status=amo.STATUS_PUBLIC) addon_updates['status'] = amo.STATUS_PUBLIC # set the modified and creation dates to match the original. addon_updates['created'] = lwt.created addon_updates['modified'] = lwt.modified addon_updates['last_updated'] = lwt.last_updated addon.update(**addon_updates) return addon
def path_prefix(self): return os.path.join(user_media_path('addons'), str(self.addon_id))
def add_static_theme_from_lwt(lwt): from olympia.activity.models import AddonLog olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID)) # Try to handle LWT with no authors author = (lwt.listed_authors or [_get_lwt_default_author()])[0] # Wrap zip in FileUpload for Addon/Version from_upload to consume. upload = FileUpload.objects.create( user=author, valid=True) destination = os.path.join( user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi') build_static_theme_xpi_from_lwt(lwt, destination) upload.update(path=destination) # Create addon + version parsed_data = parse_addon(upload, user=author) addon = Addon.initialize_addon_from_upload( parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author) addon_updates = {} # static themes are only compatible with Firefox at the moment, # not Android version = Version.from_upload( upload, addon, selected_apps=[amo.FIREFOX.id], channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data) # Set category static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get( amo.ADDON_STATICTHEME, []) lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat. lwt_category_slug = lwt_category.slug if lwt_category else 'other' static_category = static_theme_categories.get( lwt_category_slug, static_theme_categories.get('other')) AddonCategory.objects.create( addon=addon, category=Category.from_static_category(static_category, True)) # Set license lwt_license = PERSONA_LICENSES_IDS.get( lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright static_license = License.objects.get(builtin=lwt_license.builtin) version.update(license=static_license) # Set tags for addon_tag in AddonTag.objects.filter(addon=lwt): AddonTag.objects.create(addon=addon, tag=addon_tag.tag) # Steal the ratings (even with soft delete they'll be deleted anyway) addon_updates.update( average_rating=lwt.average_rating, bayesian_rating=lwt.bayesian_rating, total_ratings=lwt.total_ratings, text_ratings_count=lwt.text_ratings_count) Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version) # Modify the activity log entry too. rating_activity_log_ids = [ l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'] addonlog_qs = AddonLog.objects.filter( addon=lwt, activity_log__action__in=rating_activity_log_ids) [alog.transfer(addon) for alog in addonlog_qs.iterator()] # Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats # dashboard and future update counts, and copy the summary numbers for now. migrate_theme_update_count(lwt, addon) addon_updates.update( average_daily_users=lwt.persona.popularity or 0, hotness=lwt.persona.movers or 0) # Logging activity.log_create( amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author) # And finally sign the files (actually just one) for file_ in version.all_files: sign_file(file_) file_.update( datestatuschanged=lwt.last_updated, reviewed=datetime.now(), status=amo.STATUS_PUBLIC) addon_updates['status'] = amo.STATUS_PUBLIC # set the modified and creation dates to match the original. addon_updates['created'] = lwt.created addon_updates['modified'] = lwt.modified addon_updates['last_updated'] = lwt.last_updated addon.update(**addon_updates) return addon
def picture_dir(self): from olympia.amo.templatetags.jinja_helpers import user_media_path split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id)) return os.path.join(user_media_path('userpics'), split_id.group(2) or '0', split_id.group(1) or '0')
def gc(test_result=True): """Site-wide garbage collections.""" def days_ago(days): return datetime.today() - timedelta(days=days) log.debug('Collecting data to delete') logs = (ActivityLog.objects.filter(created__lt=days_ago(90)) .exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True)) # Paypal only keeps retrying to verify transactions for up to 3 days. If we # still have an unverified transaction after 6 days, we might as well get # rid of it. contributions_to_delete = ( Contribution.objects .filter(transaction_id__isnull=True, created__lt=days_ago(6)) .values_list('id', flat=True)) collections_to_delete = ( Collection.objects.filter(created__lt=days_ago(2), type=amo.COLLECTION_ANONYMOUS) .values_list('id', flat=True)) for chunk in chunked(logs, 100): tasks.delete_logs.delay(chunk) for chunk in chunked(contributions_to_delete, 100): tasks.delete_stale_contributions.delay(chunk) for chunk in chunked(collections_to_delete, 100): tasks.delete_anonymous_collections.delay(chunk) # Incomplete addons cannot be deleted here because when an addon is # rejected during a review it is marked as incomplete. See bug 670295. log.debug('Cleaning up test results extraction cache.') # lol at check for '/' if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/': cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name', 'validate-*', '-mtime', '+7', '-type', 'd', '-exec', 'rm', '-rf', "{}", ';') output = Popen(cmd, stdout=PIPE).communicate()[0] for line in output.split("\n"): log.debug(line) else: log.warning('MEDIA_ROOT not defined.') if user_media_path('collection_icons'): log.debug('Cleaning up uncompressed icons.') cmd = ('find', user_media_path('collection_icons'), '-name', '*__unconverted', '-mtime', '+1', '-type', 'f', '-exec', 'rm', '{}', ';') output = Popen(cmd, stdout=PIPE).communicate()[0] for line in output.split("\n"): log.debug(line) USERPICS_PATH = user_media_path('userpics') if USERPICS_PATH: log.debug('Cleaning up uncompressed userpics.') cmd = ('find', USERPICS_PATH, '-name', '*__unconverted', '-mtime', '+1', '-type', 'f', '-exec', 'rm', '{}', ';') output = Popen(cmd, stdout=PIPE).communicate()[0] for line in output.split("\n"): log.debug(line) # Delete stale FileUploads. stale_uploads = FileUpload.objects.filter( created__lte=days_ago(180)).order_by('id') for file_upload in stale_uploads: log.debug(u'[FileUpload:{uuid}] Removing file: {path}' .format(uuid=file_upload.uuid, path=file_upload.path)) if file_upload.path: try: storage.delete(file_upload.path) except OSError: pass file_upload.delete()
def get_img_dir(self): return os.path.join(user_media_path('collection_icons'), str(self.id / 1000))
def from_upload(cls, upload, addon, platforms, channel, source=None, parsed_data=None): """ Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a list of platform ids, a channel id and the parsed_data generated by parse_addon(). Note that it's the caller's responsability to ensure the file is valid. We can't check for that here because an admin may have overridden the validation results. """ assert parsed_data is not None from olympia.addons.models import AddonFeatureCompatibility if addon.status == amo.STATUS_DISABLED: raise VersionCreateError( 'Addon is Mozilla Disabled; no new versions are allowed.') license_id = None if channel == amo.RELEASE_CHANNEL_LISTED: previous_version = addon.find_latest_version( channel=channel, exclude=()) if previous_version and previous_version.license_id: license_id = previous_version.license_id version = cls.objects.create( addon=addon, version=parsed_data['version'], license_id=license_id, source=source, channel=channel, ) log.info( 'New version: %r (%s) from %r' % (version, version.id, upload)) activity.log_create(amo.LOG.ADD_VERSION, version, addon) # Update the add-on e10s compatibility since we're creating a new # version that may change that. e10s_compatibility = parsed_data.get('e10s_compatibility') if e10s_compatibility is not None: feature_compatibility = ( AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0] ) feature_compatibility.update(e10s=e10s_compatibility) compatible_apps = {} for app in parsed_data.get('apps', []): compatible_apps[app.appdata] = ApplicationsVersions( version=version, min=app.min, max=app.max, application=app.id) compatible_apps[app.appdata].save() # See #2828: sometimes when we generate the filename(s) below, in # File.from_upload(), cache-machine is confused and has trouble # fetching the ApplicationsVersions that were just created. To work # around this we pre-generate version.compatible_apps and avoid the # queries completely. version._compatible_apps = compatible_apps if addon.type in [amo.ADDON_SEARCH, amo.ADDON_STATICTHEME]: # Search extensions and static themes are always for all platforms. platforms = [amo.PLATFORM_ALL.id] else: platforms = cls._make_safe_platform_files(platforms) # Create as many files as we have platforms. Update the all_files # cached property on the Version while we're at it, because we might # need it afterwards. version.all_files = [ File.from_upload( upload, version, platform, parsed_data=parsed_data) for platform in platforms] version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW]) version.disable_old_files() # After the upload has been copied to all platforms, remove the upload. storage.delete(upload.path) version_uploaded.send(sender=version) # Generate a preview and icon for listed static themes if (addon.type == amo.ADDON_STATICTHEME and channel == amo.RELEASE_CHANNEL_LISTED): dst_root = os.path.join(user_media_path('addons'), str(addon.id)) theme_data = parsed_data.get('theme', {}) version_root = os.path.join(dst_root, unicode(version.id)) utils.extract_header_img( version.all_files[0].file_path, theme_data, version_root) preview = VersionPreview.objects.create(version=version) generate_static_theme_preview( theme_data, version_root, preview.pk) # Track the time it took from first upload through validation # (and whatever else) until a version was created. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) upload_time = now_ts - upload_start log.info('Time for version {version} creation from upload: {delta}; ' 'created={created}; now={now}' .format(delta=upload_time, version=version, created=upload.created, now=now)) statsd.timing('devhub.version_created_from_upload', upload_time) return version