def collect_all_filer_ids(verbose=False):
filer_ids = collections.defaultdict(set)
for app_name, model, object_name, fields in iter_filer_fields():
model_pgk_name = "%s.%s" % (app_name, object_name)
if verbose:
print(model_pgk_name)
for field in fields:
ids = model.objects.values_list(field.name, flat=True)
try:
ids = set(ids)
except DatabaseError as err:
log.error("Skip %s: %s", model_pgk_name, err)
continue
model_class = field.default_model_class
if isinstance(model_class, str):
model_class = load_model(model_class)
filer_ids[model_class].update(ids)
if verbose:
for model_class, ids in filer_ids.items():
print("%s: %i entries" % (model_class, len(ids)))
return filer_ids
def get_filer_attr(file_obj, attr_name):
Image = load_model(settings.FILER_IMAGE_MODEL)
if not isinstance(file_obj, Image):
raise AttributeError("'%s' is not a '%s' instance!" % (repr(file_obj), repr(Image)))
try:
return getattr(file_obj, attr_name)
except InvalidImageFormatError as err:
raise AttributeError("Can't get '%s' from '%s': %s" % (attr_name, repr(file_obj), err))
def migrate_to_filer(apps, schema_editor):
# Because filer is polymorphic, Djangos migration can't handle
from filer.utils.loader import load_model
Image = load_model(settings.FILER_IMAGE_MODEL)
VideoPlayer = apps.get_model('djangocms_video', 'VideoPlayer')
plugins = VideoPlayer.objects.all()
for video in plugins: # pragma: no cover
if video.image:
poster = Image.objects.get_or_create(
file=video.image.file, defaults={'name': video.image.name})[0]
plugins.filter(pk=video.pk).update(poster=poster)
def migrate_to_filer(apps, schema_editor):
# Because filer is polymorphic, Djangos migration can't handle
from filer.utils.loader import load_model
Image = load_model(settings.FILER_IMAGE_MODEL)
VideoPlayer = apps.get_model('djangocms_video', 'VideoPlayer')
plugins = VideoPlayer.objects.all()
for video in plugins:
if video.image:
poster = Image.objects.get_or_create(
file=video.image.file,
defaults={
'name': video.image.name
}
)[0]
plugins.filter(pk=video.pk).update(poster=poster)
def migrate_to_filer(apps, schema_editor):
# Because filer is polymorphic, Djangos migration can't handle
Image = load_model(settings.FILER_IMAGE_MODEL)
Picture = apps.get_model('djangocms_picture', 'Picture')
plugins = Picture.objects.all()
for plugin in plugins: # pragma: no cover
if plugin.image:
picture = Image.objects.get_or_create(file=plugin.image.file,
defaults={
'name':
plugin.image.name,
'default_alt_text':
plugin.alt,
'default_caption':
plugin.longdesc
})[0]
plugins.filter(pk=plugin.pk).update(picture=picture)
def create_file_obj(self,
original_filename,
file=None,
folder=None,
publish=True,
content="data",
is_public=True,
grouper=None,
mime_type="application/octet-stream",
**kwargs):
if not kwargs.get('created_by'):
kwargs['owner'] = self.superuser
if file is None:
file = self.create_file(original_filename, content)
if grouper is None:
grouper = FileGrouper.objects.create()
for filer_class in filer.settings.FILER_FILE_MODELS:
FileSubClass = load_model(filer_class)
if FileSubClass.matches_file_type(original_filename, file,
mime_type):
break
file_obj = FileSubClass.objects.create(
is_public=is_public,
original_filename=original_filename,
file=file,
folder=folder,
grouper=grouper,
mime_type=mime_type,
**kwargs)
version = create_file_version(file_obj, kwargs['owner'])
if publish:
version.publish(kwargs['owner'])
with nonversioned_manager(File):
file_obj.refresh_from_db()
return file_obj
import os
from django.conf import settings
from django.contrib.auth.models import Group
from django.core.files import File as DjangoFile
from django.test.testcases import TestCase
from tests.helpers import create_image, create_superuser
from filer import settings as filer_settings
from filer.models.clipboardmodels import Clipboard
from filer.models.foldermodels import Folder, FolderPermission
from filer.settings import FILER_IMAGE_MODEL
from filer.utils.loader import load_model
Image = load_model(FILER_IMAGE_MODEL)
class Mock():
pass
class FolderPermissionsTestCase(TestCase):
def setUp(self):
try:
from django.contrib.auth import get_user_model
User = get_user_model()
except ImportError:
from django.contrib.auth.models import User # NOQA
self.superuser = create_superuser()
self.client.login(username='******', password='******')
def migrate_text(text, user, WORKING_DIR):
images_count = 0
if '<embed' in text:
pattern = re.compile(
r'(<embed alt="([\W\w\s_][^"]+)" embedtype="image" format="(fullwidth|left|right)" id="(\d+)"/>)'
)
result = pattern.findall(text)
# Load meta data about images
image_meta_list = json.load(
open(WORKING_DIR + '/data_for_migration/images.news.json', "r"))
# Image model
Image = load_model(FILER_IMAGE_MODEL)
# Images migration
for match in list(result):
image_id = int(match[3]) # ID
meta = next(
(image
for image in image_meta_list if image["pk"] == image_id),
False)
assert meta, "PK {} was not found in the list of meta data".format(
image_id)
file_name = meta['fields']['file'].replace('original_images/', '')
file_path = WORKING_DIR + '/data_for_migration/original_images/' + file_name
# Create folder if does not exist
Folder.objects.get_or_create(
name="News images",
owner=user,
)
created_folder = Folder.objects.get(name="News images")
# Create image if does not exist
if not Image.objects.filter(name=meta['fields']['title'],
folder=created_folder).exists():
# Load image
file_obj = DjangoFile(open(file_path, 'rb'), name=file_name)
created_image = Image.objects.create(
owner=user,
folder=created_folder,
name=meta['fields']['title'],
original_filename=file_name,
file=file_obj,
is_public=True)
images_count += 1
created_image.save()
else:
created_image = Image.objects.get(name=meta['fields']['title'],
folder=created_folder)
new_image = '<a href="{src}" target="_blank"><img alt="{alt}" title="{title}" class="{cls}" src="{src}" id="{id}"/></a>'.format(
alt=match[1],
title=meta['fields']['title'],
cls='',
src=created_image.url,
id=created_image.id)
text = text.replace(match[0], new_image)
assert '<embed' not in text, "There is still embed tag in the text. \n\n {}".format(
text)
text = text.replace('<p><br/></p>', '')
text = text.replace('<br/>', '')
text = text.replace('<p class=""></p>', '')
text = text.replace('<p></p>', '')
return text, images_count
def ajax_upload(request, folder_id=None):
folder = None
path = request.POST.get('path')
path_split = path.split('/') if path else []
# check permissions and data
error_msg = None
if not request.user.is_authenticated:
# User is not logged in. Return a generic message that gives
# no data info (such as whether a folder exists or not)
error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER
elif folder_id:
try:
folder = Folder.objects.get(pk=folder_id)
except Folder.DoesNotExist:
# A folder with id=folder_id does not exist so return
# an error message specifying this
error_msg = filer.admin.clipboardadmin.NO_FOLDER_ERROR
else:
# Now check if the user has sufficient permissions to
# upload a file to the folder with id=folder_id and return
# an error message if not
no_folder_perms = (not folder.has_add_children_permission(request)
or (path and not folder.can_have_subfolders))
if no_folder_perms:
error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER
elif (not request.user.is_superuser and path_split and
not filer.settings.FILER_ALLOW_REGULAR_USERS_TO_ADD_ROOT_FOLDERS):
# If uploading the file to Unsorted Uploads (i.e. no folder_id)
# but filer is set to disallow regular users to add
# folders there and the user is not a superuser and is uploading
# folders that aren't yet created on the server (i.e.
# specifying the path param with folders that don't yet exist)
# return an error message
if not Folder.objects.filter(name=path_split[0], parent=None).exists():
error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER
if error_msg:
return JsonResponse({'error': error_msg})
try:
if len(request.FILES) == 1:
# dont check if request is ajax or not, just grab the file
upload, filename, is_raw = handle_request_files_upload(request)
else:
# else process the request as usual
upload, filename, is_raw = handle_upload(request)
# find the file type
for filer_class in filer_settings.FILER_FILE_MODELS:
FileSubClass = load_model(filer_class)
# TODO: What if there are more than one that qualify?
if FileSubClass.matches_file_type(filename, upload, request):
FileForm = modelform_factory(model=FileSubClass,
fields=('original_filename',
'owner', 'file'))
break
uploadform = FileForm(
{
'original_filename': filename,
'owner': request.user.pk
}, {'file': upload})
if uploadform.is_valid():
file_obj = uploadform.save(commit=False)
# Enforce the FILER_IS_PUBLIC_DEFAULT
file_obj.is_public = filer_settings.FILER_IS_PUBLIC_DEFAULT
# Set the file's folder
current_folder = folder
for segment in path_split:
try:
current_folder = Folder.objects.get(name=segment,
parent=current_folder)
except Folder.DoesNotExist:
# If the current_folder can't have subfolders then
# return a permission error
if current_folder and not current_folder.can_have_subfolders:
error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER
return JsonResponse({'error': error_msg})
current_folder = Folder.objects.create(
name=segment, parent=current_folder)
else:
# If the folder already exists, check the user is
# allowed to upload here
if not current_folder.has_add_children_permission(request):
error_msg = filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER
return JsonResponse({'error': error_msg})
file_obj.folder = current_folder
same_name_file_qs = get_files_distinct_grouper_queryset().annotate(
_name=NullIfEmptyStr('name'),
_original_filename=NullIfEmptyStr('original_filename'),
).annotate(
# seperate annotate is needed to get it work on python<36
# see PEP 468 for more details
_label=Coalesce('_name', '_original_filename',
Value('unnamed file')), ).filter(
folder=folder, _label=file_obj.label)
existing_file_obj = same_name_file_qs.first()
if existing_file_obj:
file_grouper = existing_file_obj.grouper
new_file_grouper = False
existing_file_version = Version.objects.get_for_content(
existing_file_obj)
if existing_file_version.state == DRAFT and not all([
existing_file_version.can_be_archived(),
existing_file_version.check_archive.as_bool(
request.user),
]):
return JsonResponse({
'error':
('Cannot archive existing {} file version'.format(
existing_file_obj))
})
else:
new_file_grouper = True
file_grouper = FileGrouper.objects.create()
file_obj.grouper = file_grouper
file_obj.save()
create_file_version(file_obj, request.user)
# Try to generate thumbnails.
if not file_obj.icons:
# There is no point to continue, as we can't generate
# thumbnails for this file. Usual reasons: bad format or
# filename.
file_obj.delete()
if new_file_grouper:
file_grouper.delete()
# This would be logged in BaseImage._generate_thumbnails()
# if FILER_ENABLE_LOGGING is on.
return JsonResponse(
{'error': 'failed to generate icons for file'},
status=500,
)
thumbnail = None
# Backwards compatibility: try to get specific icon size (32px)
# first. Then try medium icon size (they are already sorted),
# fallback to the first (smallest) configured icon.
for size in (['32'] +
filer_settings.FILER_ADMIN_ICON_SIZES[1::-1]):
try:
thumbnail = file_obj.icons[size]
break
except KeyError:
continue
data = {
'thumbnail': thumbnail,
'alt_text': '',
'label': str(file_obj),
'file_id': file_obj.pk,
'grouper_id': file_grouper.pk,
}
# prepare preview thumbnail
if type(file_obj) == Image:
thumbnail_180_options = {
'size': (180, 180),
'crop': True,
'upscale': True,
}
thumbnail_180 = file_obj.file.get_thumbnail(
thumbnail_180_options)
data['thumbnail_180'] = thumbnail_180.url
data['original_image'] = file_obj.url
return JsonResponse(data)
else:
form_errors = '; '.join([
'%s: %s' % (field, ', '.join(errors))
for field, errors in list(uploadform.errors.items())
])
raise UploadException("AJAX request not valid: form invalid '%s'" %
(form_errors, ))
except UploadException as e:
# TODO: Test
return JsonResponse({'error': str(e)}, status=500)
from django.utils.translation import ungettext_lazy, ugettext_lazy as _
from entangled.forms import EntangledModelFormMixin, EntangledModelForm
from filer.fields.image import FilerImageField, AdminImageFormField
from filer.settings import settings as filer_settings
from filer.utils.loader import load_model
from cms.plugin_pool import plugin_pool
from djangocms_text_ckeditor.fields import HTMLFormField
from cmsplugin_cascade import app_settings
from cmsplugin_cascade.fields import HiddenDictField, SizeField, MultiSizeField
from cmsplugin_cascade.models import InlineCascadeElement
from cmsplugin_cascade.plugin_base import CascadePluginBase, create_proxy_model
from cmsplugin_cascade.image import ImagePropertyMixin
from cmsplugin_cascade.utils import compute_aspect_ratio, get_image_size, parse_responsive_length
Image = load_model(filer_settings.FILER_IMAGE_MODEL)
class MarkerModelMixin(object):
@property
def data(self):
return mark_safe(json.dumps(self.glossary))
class MarkerForm(EntangledModelForm):
title = CharField(
label=_("Marker Title"),
widget=widgets.TextInput(attrs={'size': 60}),
help_text=_("Please choose a title, then go to the map to set a marker pin")
)
def ajax_upload(request, folder_id=None):
folder = None
if folder_id:
try:
# Get folder
folder = Folder.objects.get(pk=folder_id)
except Folder.DoesNotExist:
return JsonResponse({'error': filer.admin.clipboardadmin.NO_FOLDER_ERROR})
# check permissions
if folder and not folder.has_add_children_permission(request):
return JsonResponse({'error': filer.admin.clipboardadmin.NO_PERMISSIONS_FOR_FOLDER})
try:
if len(request.FILES) == 1:
# dont check if request is ajax or not, just grab the file
upload, filename, is_raw = handle_request_files_upload(request)
else:
# else process the request as usual
upload, filename, is_raw = handle_upload(request)
# find the file type
for filer_class in filer_settings.FILER_FILE_MODELS:
FileSubClass = load_model(filer_class)
# TODO: What if there are more than one that qualify?
if FileSubClass.matches_file_type(filename, upload, request):
FileForm = modelform_factory(
model=FileSubClass,
fields=('original_filename', 'owner', 'file')
)
break
uploadform = FileForm({'original_filename': filename,
'owner': request.user.pk},
{'file': upload})
if uploadform.is_valid():
file_obj = uploadform.save(commit=False)
# Enforce the FILER_IS_PUBLIC_DEFAULT
file_obj.is_public = filer_settings.FILER_IS_PUBLIC_DEFAULT
file_obj.folder = folder
same_name_file_qs = get_files_distinct_grouper_queryset().annotate(
_name=NullIfEmptyStr('name'),
_original_filename=NullIfEmptyStr('original_filename'),
).annotate(
# seperate annotate is needed to get it work on python<36
# see PEP 468 for more details
_label=Coalesce('_name', '_original_filename', Value('unnamed file')),
).filter(folder=folder, _label=file_obj.label)
existing_file_obj = same_name_file_qs.first()
if existing_file_obj:
file_grouper = existing_file_obj.grouper
new_file_grouper = False
existing_file_version = Version.objects.get_for_content(existing_file_obj)
if existing_file_version.state == DRAFT and not all([
existing_file_version.can_be_archived(),
existing_file_version.check_archive.as_bool(request.user),
]):
return JsonResponse({'error': (
'Cannot archive existing {} file version'.format(existing_file_obj)
)})
else:
new_file_grouper = True
file_grouper = FileGrouper.objects.create()
file_obj.grouper = file_grouper
file_obj.save()
create_file_version(file_obj, request.user)
# Try to generate thumbnails.
if not file_obj.icons:
# There is no point to continue, as we can't generate
# thumbnails for this file. Usual reasons: bad format or
# filename.
file_obj.delete()
if new_file_grouper:
file_grouper.delete()
# This would be logged in BaseImage._generate_thumbnails()
# if FILER_ENABLE_LOGGING is on.
return JsonResponse(
{'error': 'failed to generate icons for file'},
status=500,
)
thumbnail = None
# Backwards compatibility: try to get specific icon size (32px)
# first. Then try medium icon size (they are already sorted),
# fallback to the first (smallest) configured icon.
for size in (['32'] +
filer_settings.FILER_ADMIN_ICON_SIZES[1::-1]):
try:
thumbnail = file_obj.icons[size]
break
except KeyError:
continue
data = {
'thumbnail': thumbnail,
'alt_text': '',
'label': str(file_obj),
'file_id': file_obj.pk,
'grouper_id': file_grouper.pk,
}
# prepare preview thumbnail
if type(file_obj) == Image:
thumbnail_180_options = {
'size': (180, 180),
'crop': True,
'upscale': True,
}
thumbnail_180 = file_obj.file.get_thumbnail(
thumbnail_180_options)
data['thumbnail_180'] = thumbnail_180.url
data['original_image'] = file_obj.url
return JsonResponse(data)
else:
form_errors = '; '.join(['%s: %s' % (
field,
', '.join(errors)) for field, errors in list(
uploadform.errors.items())
])
raise UploadException(
"AJAX request not valid: form invalid '%s'" % (
form_errors,))
except UploadException as e:
return JsonResponse({'error': str(e)}, status=500)
def handle(self, *args, **options):
self.verbosity = int(options.get('verbosity'))
if settings.USE_I18N:
# e.g.: Parler models needs activated translations:
language_code = settings.LANGUAGE_CODE
self.stdout.write("activate %r translations." % language_code)
translation.activate(language_code)
self.stdout.write("settings.FILER_IMAGE_MODEL: %r" %
settings.FILER_IMAGE_MODEL)
Image = load_model(settings.FILER_IMAGE_MODEL)
self.stdout.write("Use filer Image class: %s" % repr(Image))
# image_count is not the real used image count!
image_count = Image.objects.all().count()
self.stdout.write("There are %i images in database." % image_count)
if image_count == 0:
self.stderr.write("ERROR: There are not images in database!")
sys.exit(1)
image_fallback_id = int(options.get("id"))
try:
fallback_image = Image.objects.get(pk=image_fallback_id)
except Image.DoesNotExist as err:
self.stderr.write(
"Error: Given filer instance ID %r doesn't exist!" %
image_fallback_id)
self.stderr.write("(Origin error: %s)" % err)
sys.exit(1)
if not filer_obj_exists(fallback_image, verbose=True):
self.stderr.write("Given filer image ID %r doesn't exists!" %
image_fallback_id)
sys.exit(1)
self.stdout.write("Use fallback image: %s (%s)" %
(fallback_image, fallback_image.file.name))
self.stdout.write("icons: %s" % pformat(fallback_image.icons))
self.stdout.write("thumbnails: %s" %
pformat(fallback_image.thumbnails))
#
# Count all images to calculate a accurate progressive percentage:
#
self.stdout.write("Count all images...")
next_update = time.time() + 1
total_existing_files = 0
for app_name, model, object_name, fields in iter_filer_fields():
queryset = model.objects.all()
for instance in queryset:
for field in fields:
filer_field_instance = getattr(instance, field.name)
if filer_field_instance is not None:
total_existing_files += 1
if time.time() > next_update:
# FIXME: Percent is only approximately accurate:
# image_count can be more than total_existing_files
percent = total_existing_files / image_count * 100
self.stdout.write("\t%.1f%% - %i filer files..." %
(percent, total_existing_files))
next_update = time.time() + 3
self.stdout.write("Total: %i filer files used" %
(total_existing_files))
#
# The replace broken images process:
#
processed_images = 0
total_replace_images = 0
for app_name, model, object_name, fields in iter_filer_fields():
queryset = model.objects.all()
model_count = queryset.count()
self.stdout.write("\n%i items - %s.%s" %
(model_count, app_name, model.__name__))
instance_checked = 0
existing_images = dict([(field, 0) for field in fields])
ignore_blank = existing_images.copy()
replace_images = existing_images.copy()
for instance in queryset:
save_needed = False
for field in fields:
if time.time() > next_update:
percent = processed_images / total_existing_files * 100
self.stdout.write(
"%.1f%% - %i images processed (%i images replaced)..."
%
(percent, processed_images, total_replace_images))
next_update = time.time() + 1
file_obj = getattr(instance, field.name)
if file_obj is None:
ignore_blank[field] += 1
continue
processed_images += 1
exists = filer_obj_exists(file_obj,
verbose=self.verbosity > 2)
if exists:
existing_images[field] += 1
else:
setattr(instance, field.name, fallback_image)
save_needed = True
total_replace_images += 1
replace_images[field] += 1
if save_needed:
instance.save()
instance_checked += 1
self.stdout.write("%i instanced checked:" % instance_checked)
for field in fields:
prefix = "%4i exist %4i replaced %4i ignored" % (
existing_images[field], replace_images[field],
ignore_blank[field])
self.stdout.write("%s - %s.%s.%s" % (
prefix,
app_name,
model.__name__,
field.name,
))
next_update = time.time() + 1
self.stdout.write("total:")
self.stdout.write("\tprocessed images..: %i" % processed_images)
self.stdout.write("\treplaced images...: %i" % total_replace_images)
def filer_obj_exists(file_obj, verbose=False):
if file_obj is None:
msg = "No file obj given"
if verbose:
print(msg)
else:
log.error(msg)
return False
Image = load_model(settings.FILER_IMAGE_MODEL)
if isinstance(file_obj, Image):
try:
icons = get_filer_attr(file_obj, attr_name="icons")
except AttributeError as err:
if verbose:
print(err)
else:
log.error(err)
else:
if icons:
return True
msg = "'%s' hasn't .icons" % repr(file_obj)
if verbose:
print(msg)
else:
log.error(msg)
try:
thumbnails = get_filer_attr(file_obj, attr_name="thumbnails")
except AttributeError as err:
if verbose:
print(err)
else:
log.error(err)
else:
if thumbnails:
return True
msg = "'%s' hasn't .thumbnails" % repr(file_obj)
if verbose:
print(msg)
else:
log.error(msg)
file = file_obj.file
try:
head = file.read(10)
except Exception as err:
# Catch 'Exception' here, because of differend errors, depends on
# used file storage backend.
# e.g.: IOError, AzureMissingResourceHttpError etc.
msg = "Read file error: %s" % err
if verbose:
print(msg)
else:
log.error(msg)
return False
finally:
file.close()
# print("head:", repr(head))
if len(head):
return True
msg = "file '%s' is empty" % repr(file_obj)
if verbose:
print(msg)
else:
log.error(msg)
return False
def clean(self, value):
Image = load_model(filer.settings.FILER_IMAGE_MODEL)
self.queryset = self.queryset.filter(
files__in=File._base_manager.instance_of(Image)).distinct()
value = super().clean(value)
return value
