def collect_all_filer_ids(verbose=False): filer_ids = collections.defaultdict(set) for app_name, model, object_name, fields in iter_filer_fields(): model_pgk_name = "%s.%s" % (app_name, object_name) if verbose: print(model_pgk_name) for field in fields: ids = model.objects.values_list(field.name, flat=True) try: ids = set(ids) except DatabaseError as err: log.error("Skip %s: %s", model_pgk_name, err) continue model_class = field.default_model_class if isinstance(model_class, str): model_class = load_model(model_class) filer_ids[model_class].update(ids) if verbose: for model_class, ids in filer_ids.items(): print("%s: %i entries" % (model_class, len(ids))) return filer_ids
def get_filer_attr(file_obj, attr_name): Image = load_model(settings.FILER_IMAGE_MODEL) if not isinstance(file_obj, Image): raise AttributeError("'%s' is not a '%s' instance!" % (repr(file_obj), repr(Image))) try: return getattr(file_obj, attr_name) except InvalidImageFormatError as err: raise AttributeError("Can't get '%s' from '%s': %s" % (attr_name, repr(file_obj), err))
def migrate_to_filer(apps, schema_editor): # Because filer is polymorphic, Djangos migration can't handle from filer.utils.loader import load_model Image = load_model(settings.FILER_IMAGE_MODEL) VideoPlayer = apps.get_model('djangocms_video', 'VideoPlayer') plugins = VideoPlayer.objects.all() for video in plugins: # pragma: no cover if video.image: poster = Image.objects.get_or_create( file=video.image.file, defaults={'name': video.image.name})[0] plugins.filter(pk=video.pk).update(poster=poster)
def migrate_to_filer(apps, schema_editor): # Because filer is polymorphic, Djangos migration can't handle from filer.utils.loader import load_model Image = load_model(settings.FILER_IMAGE_MODEL) VideoPlayer = apps.get_model('djangocms_video', 'VideoPlayer') plugins = VideoPlayer.objects.all() for video in plugins: if video.image: poster = Image.objects.get_or_create( file=video.image.file, defaults={ 'name': video.image.name } )[0] plugins.filter(pk=video.pk).update(poster=poster)
def migrate_to_filer(apps, schema_editor): # Because filer is polymorphic, Djangos migration can't handle Image = load_model(settings.FILER_IMAGE_MODEL) Picture = apps.get_model('djangocms_picture', 'Picture') plugins = Picture.objects.all() for plugin in plugins: # pragma: no cover if plugin.image: picture = Image.objects.get_or_create(file=plugin.image.file, defaults={ 'name': plugin.image.name, 'default_alt_text': plugin.alt, 'default_caption': plugin.longdesc })[0] plugins.filter(pk=plugin.pk).update(picture=picture)
def create_file_obj(self, original_filename, file=None, folder=None, publish=True, content="data", is_public=True, grouper=None, mime_type="application/octet-stream", **kwargs): if not kwargs.get('created_by'): kwargs['owner'] = self.superuser if file is None: file = self.create_file(original_filename, content) if grouper is None: grouper = FileGrouper.objects.create() for filer_class in filer.settings.FILER_FILE_MODELS: FileSubClass = load_model(filer_class) if FileSubClass.matches_file_type(original_filename, file, mime_type): break file_obj = FileSubClass.objects.create( is_public=is_public, original_filename=original_filename, file=file, folder=folder, grouper=grouper, mime_type=mime_type, **kwargs) version = create_file_version(file_obj, kwargs['owner']) if publish: version.publish(kwargs['owner']) with nonversioned_manager(File): file_obj.refresh_from_db() return file_obj
import os from django.conf import settings from django.contrib.auth.models import Group from django.core.files import File as DjangoFile from django.test.testcases import TestCase from tests.helpers import create_image, create_superuser from filer import settings as filer_settings from filer.models.clipboardmodels import Clipboard from filer.models.foldermodels import Folder, FolderPermission from filer.settings import FILER_IMAGE_MODEL from filer.utils.loader import load_model Image = load_model(FILER_IMAGE_MODEL) class Mock(): pass class FolderPermissionsTestCase(TestCase): def setUp(self): try: from django.contrib.auth import get_user_model User = get_user_model() except ImportError: from django.contrib.auth.models import User # NOQA self.superuser = create_superuser() self.client.login(username='******', password='******')
def migrate_text(text, user, WORKING_DIR): images_count = 0 if '<embed' in text: pattern = re.compile( r'(<embed alt="([\W\w\s_][^"]+)" embedtype="image" format="(fullwidth|left|right)" id="(\d+)"/>)' ) result = pattern.findall(text) # Load meta data about images image_meta_list = json.load( open(WORKING_DIR + '/data_for_migration/images.news.json', "r")) # Image model Image = load_model(FILER_IMAGE_MODEL) # Images migration for match in list(result): image_id = int(match[3]) # ID meta = next( (image for image in image_meta_list if image["pk"] == image_id), False) assert meta, "PK {} was not found in the list of meta data".format( image_id) file_name = meta['fields']['file'].replace('original_images/', '') file_path = WORKING_DIR + '/data_for_migration/original_images/' + file_name # Create folder if does not exist Folder.objects.get_or_create( name="News images", owner=user, ) created_folder = Folder.objects.get(name="News images") # Create image if does not exist if not Image.objects.filter(name=meta['fields']['title'], folder=created_folder).exists(): # Load image file_obj = DjangoFile(open(file_path, 'rb'), name=file_name) created_image = Image.objects.create( owner=user, folder=created_folder, name=meta['fields']['title'], original_filename=file_name, file=file_obj, is_public=True) images_count += 1 created_image.save() else: created_image = Image.objects.get(name=meta['fields']['title'], folder=created_folder) new_image = '<a href="{src}" target="_blank"><img alt="{alt}" title="{title}" class="{cls}" src="{src}" id="{id}"/></a>'.format( alt=match[1], title=meta['fields']['title'], cls='', src=created_image.url, id=created_image.id) text = text.replace(match[0], new_image) assert '<embed' not in text, "There is still embed tag in the text. \n\n {}".format( text) text = text.replace('<p><br/></p>', '') text = text.replace('<br/>', '') text = text.replace('<p class=""></p>', '') text = text.replace('<p></p>', '') return text, images_count
def ajax_upload(request, folder_id=None): folder = None path = request.POST.get('path') path_split = path.split('/') if path else [] # check permissions and data error_msg = None if not request.user.is_authenticated: # User is not logged in. Return a generic message that gives # no data info (such as whether a folder exists or not) error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER elif folder_id: try: folder = Folder.objects.get(pk=folder_id) except Folder.DoesNotExist: # A folder with id=folder_id does not exist so return # an error message specifying this error_msg = filer.admin.clipboardadmin.NO_FOLDER_ERROR else: # Now check if the user has sufficient permissions to # upload a file to the folder with id=folder_id and return # an error message if not no_folder_perms = (not folder.has_add_children_permission(request) or (path and not folder.can_have_subfolders)) if no_folder_perms: error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER elif (not request.user.is_superuser and path_split and not filer.settings.FILER_ALLOW_REGULAR_USERS_TO_ADD_ROOT_FOLDERS): # If uploading the file to Unsorted Uploads (i.e. no folder_id) # but filer is set to disallow regular users to add # folders there and the user is not a superuser and is uploading # folders that aren't yet created on the server (i.e. # specifying the path param with folders that don't yet exist) # return an error message if not Folder.objects.filter(name=path_split[0], parent=None).exists(): error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER if error_msg: return JsonResponse({'error': error_msg}) try: if len(request.FILES) == 1: # dont check if request is ajax or not, just grab the file upload, filename, is_raw = handle_request_files_upload(request) else: # else process the request as usual upload, filename, is_raw = handle_upload(request) # find the file type for filer_class in filer_settings.FILER_FILE_MODELS: FileSubClass = load_model(filer_class) # TODO: What if there are more than one that qualify? if FileSubClass.matches_file_type(filename, upload, request): FileForm = modelform_factory(model=FileSubClass, fields=('original_filename', 'owner', 'file')) break uploadform = FileForm( { 'original_filename': filename, 'owner': request.user.pk }, {'file': upload}) if uploadform.is_valid(): file_obj = uploadform.save(commit=False) # Enforce the FILER_IS_PUBLIC_DEFAULT file_obj.is_public = filer_settings.FILER_IS_PUBLIC_DEFAULT # Set the file's folder current_folder = folder for segment in path_split: try: current_folder = Folder.objects.get(name=segment, parent=current_folder) except Folder.DoesNotExist: # If the current_folder can't have subfolders then # return a permission error if current_folder and not current_folder.can_have_subfolders: error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER return JsonResponse({'error': error_msg}) current_folder = Folder.objects.create( name=segment, parent=current_folder) else: # If the folder already exists, check the user is # allowed to upload here if not current_folder.has_add_children_permission(request): error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER return JsonResponse({'error': error_msg}) file_obj.folder = current_folder same_name_file_qs = get_files_distinct_grouper_queryset().annotate( _name=NullIfEmptyStr('name'), _original_filename=NullIfEmptyStr('original_filename'), ).annotate( # seperate annotate is needed to get it work on python<36 # see PEP 468 for more details _label=Coalesce('_name', '_original_filename', Value('unnamed file')), ).filter( folder=folder, _label=file_obj.label) existing_file_obj = same_name_file_qs.first() if existing_file_obj: file_grouper = existing_file_obj.grouper new_file_grouper = False existing_file_version = Version.objects.get_for_content( existing_file_obj) if existing_file_version.state == DRAFT and not all([ existing_file_version.can_be_archived(), existing_file_version.check_archive.as_bool( request.user), ]): return JsonResponse({ 'error': ('Cannot archive existing {} file version'.format( existing_file_obj)) }) else: new_file_grouper = True file_grouper = FileGrouper.objects.create() file_obj.grouper = file_grouper file_obj.save() create_file_version(file_obj, request.user) # Try to generate thumbnails. if not file_obj.icons: # There is no point to continue, as we can't generate # thumbnails for this file. Usual reasons: bad format or # filename. file_obj.delete() if new_file_grouper: file_grouper.delete() # This would be logged in BaseImage._generate_thumbnails() # if FILER_ENABLE_LOGGING is on. return JsonResponse( {'error': 'failed to generate icons for file'}, status=500, ) thumbnail = None # Backwards compatibility: try to get specific icon size (32px) # first. Then try medium icon size (they are already sorted), # fallback to the first (smallest) configured icon. for size in (['32'] + filer_settings.FILER_ADMIN_ICON_SIZES[1::-1]): try: thumbnail = file_obj.icons[size] break except KeyError: continue data = { 'thumbnail': thumbnail, 'alt_text': '', 'label': str(file_obj), 'file_id': file_obj.pk, 'grouper_id': file_grouper.pk, } # prepare preview thumbnail if type(file_obj) == Image: thumbnail_180_options = { 'size': (180, 180), 'crop': True, 'upscale': True, } thumbnail_180 = file_obj.file.get_thumbnail( thumbnail_180_options) data['thumbnail_180'] = thumbnail_180.url data['original_image'] = file_obj.url return JsonResponse(data) else: form_errors = '; '.join([ '%s: %s' % (field, ', '.join(errors)) for field, errors in list(uploadform.errors.items()) ]) raise UploadException("AJAX request not valid: form invalid '%s'" % (form_errors, )) except UploadException as e: # TODO: Test return JsonResponse({'error': str(e)}, status=500)
from django.utils.translation import ungettext_lazy, ugettext_lazy as _ from entangled.forms import EntangledModelFormMixin, EntangledModelForm from filer.fields.image import FilerImageField, AdminImageFormField from filer.settings import settings as filer_settings from filer.utils.loader import load_model from cms.plugin_pool import plugin_pool from djangocms_text_ckeditor.fields import HTMLFormField from cmsplugin_cascade import app_settings from cmsplugin_cascade.fields import HiddenDictField, SizeField, MultiSizeField from cmsplugin_cascade.models import InlineCascadeElement from cmsplugin_cascade.plugin_base import CascadePluginBase, create_proxy_model from cmsplugin_cascade.image import ImagePropertyMixin from cmsplugin_cascade.utils import compute_aspect_ratio, get_image_size, parse_responsive_length Image = load_model(filer_settings.FILER_IMAGE_MODEL) class MarkerModelMixin(object): @property def data(self): return mark_safe(json.dumps(self.glossary)) class MarkerForm(EntangledModelForm): title = CharField( label=_("Marker Title"), widget=widgets.TextInput(attrs={'size': 60}), help_text=_("Please choose a title, then go to the map to set a marker pin") )
def ajax_upload(request, folder_id=None): folder = None if folder_id: try: # Get folder folder = Folder.objects.get(pk=folder_id) except Folder.DoesNotExist: return JsonResponse({'error': filer.admin.clipboardadmin.NO_FOLDER_ERROR}) # check permissions if folder and not folder.has_add_children_permission(request): return JsonResponse({'error': filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER}) try: if len(request.FILES) == 1: # dont check if request is ajax or not, just grab the file upload, filename, is_raw = handle_request_files_upload(request) else: # else process the request as usual upload, filename, is_raw = handle_upload(request) # find the file type for filer_class in filer_settings.FILER_FILE_MODELS: FileSubClass = load_model(filer_class) # TODO: What if there are more than one that qualify? if FileSubClass.matches_file_type(filename, upload, request): FileForm = modelform_factory( model=FileSubClass, fields=('original_filename', 'owner', 'file') ) break uploadform = FileForm({'original_filename': filename, 'owner': request.user.pk}, {'file': upload}) if uploadform.is_valid(): file_obj = uploadform.save(commit=False) # Enforce the FILER_IS_PUBLIC_DEFAULT file_obj.is_public = filer_settings.FILER_IS_PUBLIC_DEFAULT file_obj.folder = folder same_name_file_qs = get_files_distinct_grouper_queryset().annotate( _name=NullIfEmptyStr('name'), _original_filename=NullIfEmptyStr('original_filename'), ).annotate( # seperate annotate is needed to get it work on python<36 # see PEP 468 for more details _label=Coalesce('_name', '_original_filename', Value('unnamed file')), ).filter(folder=folder, _label=file_obj.label) existing_file_obj = same_name_file_qs.first() if existing_file_obj: file_grouper = existing_file_obj.grouper new_file_grouper = False existing_file_version = Version.objects.get_for_content(existing_file_obj) if existing_file_version.state == DRAFT and not all([ existing_file_version.can_be_archived(), existing_file_version.check_archive.as_bool(request.user), ]): return JsonResponse({'error': ( 'Cannot archive existing {} file version'.format(existing_file_obj) )}) else: new_file_grouper = True file_grouper = FileGrouper.objects.create() file_obj.grouper = file_grouper file_obj.save() create_file_version(file_obj, request.user) # Try to generate thumbnails. if not file_obj.icons: # There is no point to continue, as we can't generate # thumbnails for this file. Usual reasons: bad format or # filename. file_obj.delete() if new_file_grouper: file_grouper.delete() # This would be logged in BaseImage._generate_thumbnails() # if FILER_ENABLE_LOGGING is on. return JsonResponse( {'error': 'failed to generate icons for file'}, status=500, ) thumbnail = None # Backwards compatibility: try to get specific icon size (32px) # first. Then try medium icon size (they are already sorted), # fallback to the first (smallest) configured icon. for size in (['32'] + filer_settings.FILER_ADMIN_ICON_SIZES[1::-1]): try: thumbnail = file_obj.icons[size] break except KeyError: continue data = { 'thumbnail': thumbnail, 'alt_text': '', 'label': str(file_obj), 'file_id': file_obj.pk, 'grouper_id': file_grouper.pk, } # prepare preview thumbnail if type(file_obj) == Image: thumbnail_180_options = { 'size': (180, 180), 'crop': True, 'upscale': True, } thumbnail_180 = file_obj.file.get_thumbnail( thumbnail_180_options) data['thumbnail_180'] = thumbnail_180.url data['original_image'] = file_obj.url return JsonResponse(data) else: form_errors = '; '.join(['%s: %s' % ( field, ', '.join(errors)) for field, errors in list( uploadform.errors.items()) ]) raise UploadException( "AJAX request not valid: form invalid '%s'" % ( form_errors,)) except UploadException as e: return JsonResponse({'error': str(e)}, status=500)
def handle(self, *args, **options): self.verbosity = int(options.get('verbosity')) if settings.USE_I18N: # e.g.: Parler models needs activated translations: language_code = settings.LANGUAGE_CODE self.stdout.write("activate %r translations." % language_code) translation.activate(language_code) self.stdout.write("settings.FILER_IMAGE_MODEL: %r" % settings.FILER_IMAGE_MODEL) Image = load_model(settings.FILER_IMAGE_MODEL) self.stdout.write("Use filer Image class: %s" % repr(Image)) # image_count is not the real used image count! image_count = Image.objects.all().count() self.stdout.write("There are %i images in database." % image_count) if image_count == 0: self.stderr.write("ERROR: There are not images in database!") sys.exit(1) image_fallback_id = int(options.get("id")) try: fallback_image = Image.objects.get(pk=image_fallback_id) except Image.DoesNotExist as err: self.stderr.write( "Error: Given filer instance ID %r doesn't exist!" % image_fallback_id) self.stderr.write("(Origin error: %s)" % err) sys.exit(1) if not filer_obj_exists(fallback_image, verbose=True): self.stderr.write("Given filer image ID %r doesn't exists!" % image_fallback_id) sys.exit(1) self.stdout.write("Use fallback image: %s (%s)" % (fallback_image, fallback_image.file.name)) self.stdout.write("icons: %s" % pformat(fallback_image.icons)) self.stdout.write("thumbnails: %s" % pformat(fallback_image.thumbnails)) # # Count all images to calculate a accurate progressive percentage: # self.stdout.write("Count all images...") next_update = time.time() + 1 total_existing_files = 0 for app_name, model, object_name, fields in iter_filer_fields(): queryset = model.objects.all() for instance in queryset: for field in fields: filer_field_instance = getattr(instance, field.name) if filer_field_instance is not None: total_existing_files += 1 if time.time() > next_update: # FIXME: Percent is only approximately accurate: # image_count can be more than total_existing_files percent = total_existing_files / image_count * 100 self.stdout.write("\t%.1f%% - %i filer files..." % (percent, total_existing_files)) next_update = time.time() + 3 self.stdout.write("Total: %i filer files used" % (total_existing_files)) # # The replace broken images process: # processed_images = 0 total_replace_images = 0 for app_name, model, object_name, fields in iter_filer_fields(): queryset = model.objects.all() model_count = queryset.count() self.stdout.write("\n%i items - %s.%s" % (model_count, app_name, model.__name__)) instance_checked = 0 existing_images = dict([(field, 0) for field in fields]) ignore_blank = existing_images.copy() replace_images = existing_images.copy() for instance in queryset: save_needed = False for field in fields: if time.time() > next_update: percent = processed_images / total_existing_files * 100 self.stdout.write( "%.1f%% - %i images processed (%i images replaced)..." % (percent, processed_images, total_replace_images)) next_update = time.time() + 1 file_obj = getattr(instance, field.name) if file_obj is None: ignore_blank[field] += 1 continue processed_images += 1 exists = filer_obj_exists(file_obj, verbose=self.verbosity > 2) if exists: existing_images[field] += 1 else: setattr(instance, field.name, fallback_image) save_needed = True total_replace_images += 1 replace_images[field] += 1 if save_needed: instance.save() instance_checked += 1 self.stdout.write("%i instanced checked:" % instance_checked) for field in fields: prefix = "%4i exist %4i replaced %4i ignored" % ( existing_images[field], replace_images[field], ignore_blank[field]) self.stdout.write("%s - %s.%s.%s" % ( prefix, app_name, model.__name__, field.name, )) next_update = time.time() + 1 self.stdout.write("total:") self.stdout.write("\tprocessed images..: %i" % processed_images) self.stdout.write("\treplaced images...: %i" % total_replace_images)
def filer_obj_exists(file_obj, verbose=False): if file_obj is None: msg = "No file obj given" if verbose: print(msg) else: log.error(msg) return False Image = load_model(settings.FILER_IMAGE_MODEL) if isinstance(file_obj, Image): try: icons = get_filer_attr(file_obj, attr_name="icons") except AttributeError as err: if verbose: print(err) else: log.error(err) else: if icons: return True msg = "'%s' hasn't .icons" % repr(file_obj) if verbose: print(msg) else: log.error(msg) try: thumbnails = get_filer_attr(file_obj, attr_name="thumbnails") except AttributeError as err: if verbose: print(err) else: log.error(err) else: if thumbnails: return True msg = "'%s' hasn't .thumbnails" % repr(file_obj) if verbose: print(msg) else: log.error(msg) file = file_obj.file try: head = file.read(10) except Exception as err: # Catch 'Exception' here, because of differend errors, depends on # used file storage backend. # e.g.: IOError, AzureMissingResourceHttpError etc. msg = "Read file error: %s" % err if verbose: print(msg) else: log.error(msg) return False finally: file.close() # print("head:", repr(head)) if len(head): return True msg = "file '%s' is empty" % repr(file_obj) if verbose: print(msg) else: log.error(msg) return False
def clean(self, value): Image = load_model(filer.settings.FILER_IMAGE_MODEL) self.queryset = self.queryset.filter( files__in=File._base_manager.instance_of(Image)).distinct() value = super().clean(value) return value