class Signature(models.Model):
user = models.OneToOneField(settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
verbose_name=_("User"))
signature = models.ImageField(null=True,
blank=True,
upload_to=signature_path,
storage=HashedFilenameStorage())
timestamp = models.DateTimeField(default=timezone.now)
class Meta:
verbose_name = _('Signature')
verbose_name_plural = _('Signatures')
def __str__(self):
return str(self.user)
def remove_signature_file(self):
if self.signature and os.path.exists(self.signature.path):
os.remove(self.signature.path)
def get_signature_bytes(self):
if not self.signature:
return None
try:
self.signature.open()
except IOError:
# File was deleted, set field to None
self.signature = None
self.save()
return None
try:
return self.signature.read()
finally:
self.signature.close()
def handle(self, *args, **options):
translation.activate(settings.LANGUAGE_CODE)
if not hasattr(os, 'scandir'):
raise NotImplemented('Requires Python 3.5+')
self.storage = HashedFilenameStorage()
self.directory = options['directory']
self.run_all = options.get('run_all') == 'all'
self.run_now = options.get('run_now') == 'run-now'
self.missing_attachments = []
if not self.run_now:
print('DRY RUN MODE')
else:
print('Running for real now')
for folder in os.scandir(self.directory):
if folder.is_dir():
self.handle_folder(folder)
with open('missing_attachments.json', 'w') as f:
json.dump(self.missing_attachments, f)
def handle(self, *args, **options):
translation.activate(settings.LANGUAGE_CODE)
if not hasattr(os, 'scandir'):
raise NotImplementedError('Requires Python 3.5+')
self.storage = HashedFilenameStorage()
self.directory = options['directory']
self.run_all = options.get('run_all') == 'all'
self.run_now = options.get('run_now') == 'run-now'
self.missing_attachments = []
if not self.run_now:
print('DRY RUN MODE')
else:
print('Running for real now')
for folder in os.scandir(self.directory):
if folder.is_dir():
self.handle_folder(folder)
with open('missing_attachments.json', 'w') as f:
json.dump(self.missing_attachments, f)
class FoiAttachment(models.Model):
belongs_to = models.ForeignKey(FoiMessage,
null=True,
verbose_name=_("Belongs to message"),
on_delete=models.CASCADE,
related_name='foiattachment_set')
name = models.CharField(_("Name"), max_length=255)
file = models.FileField(_("File"),
upload_to=upload_to,
max_length=255,
storage=HashedFilenameStorage(),
db_index=True)
size = models.IntegerField(_("Size"), blank=True, null=True)
filetype = models.CharField(_("File type"), blank=True, max_length=100)
format = models.CharField(_("Format"), blank=True, max_length=100)
can_approve = models.BooleanField(_("User can approve"), default=True)
approved = models.BooleanField(_("Approved"), default=False)
redacted = models.ForeignKey('self',
verbose_name=_("Redacted Version"),
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name='unredacted_set')
is_redacted = models.BooleanField(_("Is redacted"), default=False)
converted = models.ForeignKey('self',
verbose_name=_("Converted Version"),
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name='original_set')
is_converted = models.BooleanField(_("Is converted"), default=False)
timestamp = models.DateTimeField(null=True, default=timezone.now)
pending = models.BooleanField(default=False)
document = models.OneToOneField(Document,
null=True,
blank=True,
related_name='attachment',
on_delete=models.SET_NULL)
objects = FoiAttachmentManager()
attachment_published = Signal(providing_args=['user'])
attachment_deleted = Signal(providing_args=['user'])
attachment_redacted = Signal(providing_args=['user'])
document_created = Signal(providing_args=['user'])
class Meta:
ordering = ('name', )
unique_together = (("belongs_to", "name"), )
# order_with_respect_to = 'belongs_to'
verbose_name = _('Attachment')
verbose_name_plural = _('Attachments')
def __str__(self):
return "%s (%s) of %s" % (self.name, self.size, self.belongs_to)
def index_content(self):
return "\n".join((self.name, ))
def get_html_id(self):
return _("attachment-%(id)d") % {"id": self.id}
def get_bytes(self):
self.file.open(mode='rb')
try:
return self.file.read()
finally:
self.file.close()
@property
def can_redact(self):
return self.redacted is not None or (self.can_approve and self.is_pdf)
@property
def can_delete(self):
if not self.belongs_to.is_postal:
return False
if not self.can_approve:
return False
now = timezone.now()
return self.timestamp > (now - DELETE_TIMEFRAME)
@property
def can_edit(self):
return self.can_redact or self.can_delete or self.can_approve
@property
def can_link(self):
return self.approved or not (self.can_redact and self.can_approve)
@property
def is_pdf(self):
return self.filetype in PDF_FILETYPES or (
self.name and self.name.endswith(('.pdf', '.PDF'))
and self.filetype == 'application/octet-stream')
@property
def is_word(self):
return self.filetype in WORD_FILETYPES or (
self.name and self.name.endswith(WORD_FILEEXTENSIONS))
@property
def is_excel(self):
return self.filetype in EXCEL_FILETYPES or (
self.name and self.name.endswith(EXCEL_FILEEXTENSIONS))
@property
def is_text(self):
return self.filetype.startswith('text/')
@property
def is_archive(self):
return self.filetype in ARCHIVE_FILETYPES or (
self.name and self.name.endswith(ARCHIVE_FILEEXTENSIONS))
@property
def is_powerpoint(self):
return self.filetype in POWERPOINT_FILETYPES or (
self.name and self.name.endswith(POWERPOINT_FILETEXTENSIONS))
@property
def is_image(self):
return (self.filetype.startswith('image/')
or self.filetype in IMAGE_FILETYPES or self.name.endswith(
('.jpg', '.jpeg', '.gif', '.png')))
@property
def is_mail_decoration(self):
return self.is_image and self.size and self.size < 1024 * 60
@property
def is_irrelevant(self):
return self.is_mail_decoration or self.is_signature
@property
def is_signature(self):
return self.name.endswith(
('.p7s', '.vcf', '.asc')) and self.size < 1024 * 15
@property
def can_embed(self):
return self.filetype in EMBEDDABLE_FILETYPES or self.is_pdf
def get_anchor_url(self):
if self.belongs_to:
return self.belongs_to.get_absolute_url()
return '#' + self.get_html_id()
def get_domain_anchor_url(self):
return '%s%s' % (settings.SITE_URL, self.get_anchor_url())
def get_absolute_url(self):
fr = self.belongs_to.request
return reverse('foirequest-show_attachment',
kwargs={
'slug': fr.slug,
'message_id': self.belongs_to.pk,
'attachment_name': self.name
})
def get_file_url(self):
'''
Hook method for django-filingcabinet
'''
return self.get_absolute_domain_file_url()
def get_file_path(self):
return self.file.path
def get_crossdomain_auth(self):
from ..auth import AttachmentCrossDomainMediaAuth
return AttachmentCrossDomainMediaAuth({
'object': self,
})
def send_internal_file(self):
return self.get_crossdomain_auth().send_internal_file()
def get_absolute_domain_url(self):
return '%s%s' % (settings.SITE_URL, self.get_absolute_url())
def get_absolute_domain_auth_url(self):
return self.get_crossdomain_auth().get_full_auth_url()
def get_authorized_absolute_domain_file_url(self):
return self.get_absolute_domain_file_url(authorized=True)
def get_absolute_domain_file_url(self, authorized=False):
return self.get_crossdomain_auth().get_full_media_url(
authorized=authorized)
def approve_and_save(self):
self.approved = True
self.save()
if self.document:
foirequest = self.belongs_to.request
should_be_public = foirequest.public
if self.document.public != should_be_public:
self.document.public = should_be_public
self.document.save()
def remove_file_and_delete(self):
if self.file:
other_references = FoiAttachment.objects.filter(
file=self.file.name).exclude(id=self.id).exists()
if not other_references:
self.file.delete(save=False)
self.delete()
def can_convert_to_pdf(self):
ft = self.filetype.lower()
name = self.name.lower()
return (self.converted_id is None
and can_convert_to_pdf(ft, name=name))
def create_document(self, title=None):
if self.document is not None:
return self.document
if not self.is_pdf:
return
if self.converted_id or self.redacted_id:
return
foirequest = self.belongs_to.request
doc = Document.objects.create(
original=self,
user=foirequest.user,
public=foirequest.public,
title=title or self.name,
foirequest=self.belongs_to.request,
pending=True,
publicbody=self.belongs_to.sender_public_body)
self.document = doc
self.save()
return doc
class User(AbstractBaseUser, PermissionsMixin):
username_validator = UnicodeUsernameValidator()
username = models.CharField(
_('username'),
max_length=150,
unique=True,
help_text=
_('Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.'
),
validators=[username_validator],
error_messages={
'unique': _("A user with that username already exists."),
},
)
first_name = models.CharField(_('first name'), max_length=30, blank=True)
last_name = models.CharField(_('last name'), max_length=150, blank=True)
email = models.EmailField(_('email address'),
unique=True,
null=True,
blank=True)
is_staff = models.BooleanField(
_('staff status'),
default=False,
help_text=_(
'Designates whether the user can log into this admin site.'),
)
is_active = models.BooleanField(
_('active'),
default=True,
help_text=_(
'Designates whether this user should be treated as active. '
'Unselect this instead of deleting accounts.'),
)
date_joined = models.DateTimeField(_('date joined'), default=timezone.now)
organization = models.CharField(_('Organization'),
blank=True,
max_length=255)
organization_url = models.URLField(_('Organization URL'),
blank=True,
max_length=255)
private = models.BooleanField(_('Private'), default=False)
address = models.TextField(_('Address'), blank=True)
terms = models.BooleanField(_('Accepted Terms'), default=True)
newsletter = models.BooleanField(_('Wants Newsletter'), default=False)
profile_text = models.TextField(blank=True)
profile_photo = models.ImageField(null=True,
blank=True,
upload_to=profile_photo_path,
storage=HashedFilenameStorage())
is_trusted = models.BooleanField(_('Trusted'), default=False)
is_blocked = models.BooleanField(_('Blocked'), default=False)
is_deleted = models.BooleanField(
_('deleted'),
default=False,
help_text=_('Designates whether this user was deleted.'))
date_left = models.DateTimeField(_('date left'),
default=None,
null=True,
blank=True)
objects = UserManager()
USERNAME_FIELD = 'email'
EMAIL_FIELD = 'email'
REQUIRED_FIELDS = ['username']
def __str__(self):
if self.email is None:
return self.username
return self.email
def get_absolute_url(self):
if self.private:
return ""
return reverse('account-profile', kwargs={'slug': self.username})
def get_full_name(self):
"""
Returns the first_name plus the last_name, with a space in between.
"""
full_name = '%s %s' % (self.first_name, self.last_name)
return full_name.strip()
def get_short_name(self):
"Returns the short name for the user."
return self.first_name
def get_dict(self, fields):
d = get_dict(self, fields)
d['request_count'] = self.foirequest_set.all().count()
return d
def trusted(self):
return self.is_trusted or self.is_staff or self.is_superuser
def show_newsletter(self):
return has_newsletter() and not self.newsletter
@classmethod
def export_csv(cls, queryset):
fields = (
"id",
"first_name",
"last_name",
"email",
"organization",
"organization_url",
"private",
"date_joined",
"is_staff",
"address",
"terms",
"newsletter",
"request_count",
)
return export_csv(queryset, fields)
def as_json(self):
return json.dumps({
'id': self.id,
'first_name': self.first_name,
'last_name': self.last_name,
'address': self.address,
'private': self.private,
'email': self.email,
'organization': self.organization
})
def display_name(self):
if self.private:
return str(_("<< Name Not Public >>"))
else:
if self.organization:
return '%s (%s)' % (self.get_full_name(), self.organization)
else:
return self.get_full_name()
def get_account_service(self):
from .services import AccountService
return AccountService(self)
def get_autologin_url(self, url):
from .services import AccountService
service = AccountService(self)
return service.get_autologin_url(url)
def get_password_change_form(self, *args, **kwargs):
from .forms import SetPasswordForm
return SetPasswordForm(self, *args, **kwargs)
def get_change_form(self, *args, **kwargs):
from .forms import UserChangeForm
return UserChangeForm(self, *args, **kwargs)
class FoiAttachment(models.Model):
belongs_to = models.ForeignKey(FoiMessage,
null=True,
verbose_name=_("Belongs to request"),
on_delete=models.CASCADE,
related_name='foiattachment_set')
name = models.CharField(_("Name"), max_length=255)
file = models.FileField(_("File"),
upload_to=upload_to,
max_length=255,
storage=HashedFilenameStorage(),
db_index=True)
size = models.IntegerField(_("Size"), blank=True, null=True)
filetype = models.CharField(_("File type"), blank=True, max_length=100)
format = models.CharField(_("Format"), blank=True, max_length=100)
can_approve = models.BooleanField(_("User can approve"), default=True)
approved = models.BooleanField(_("Approved"), default=False)
redacted = models.ForeignKey('self',
verbose_name=_("Redacted Version"),
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name='unredacted_set')
is_redacted = models.BooleanField(_("Is redacted"), default=False)
converted = models.ForeignKey('self',
verbose_name=_("Converted Version"),
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name='original_set')
is_converted = models.BooleanField(_("Is converted"), default=False)
timestamp = models.DateTimeField(null=True, default=timezone.now)
document = models.OneToOneField(Document,
null=True,
blank=True,
related_name='attachment',
on_delete=models.SET_NULL)
attachment_published = Signal(providing_args=[])
class Meta:
ordering = ('name', )
unique_together = (("belongs_to", "name"), )
# order_with_respect_to = 'belongs_to'
verbose_name = _('Attachment')
verbose_name_plural = _('Attachments')
def __str__(self):
return "%s (%s) of %s" % (self.name, self.size, self.belongs_to)
def index_content(self):
return "\n".join((self.name, ))
def get_html_id(self):
return _("attachment-%(id)d") % {"id": self.id}
def get_internal_url(self):
return settings.FOI_MEDIA_URL + self.file.name
def get_bytes(self):
self.file.open(mode='rb')
try:
return self.file.read()
finally:
self.file.close()
@property
def can_redact(self):
return can_redact_file(self.filetype, name=self.name)
@property
def can_delete(self):
if not self.belongs_to.is_postal:
return False
if not self.can_approve:
return False
now = timezone.now()
return self.timestamp > (now - DELETE_TIMEFRAME)
@property
def pending(self):
return not self.file
@property
def can_edit(self):
return self.can_redact or self.can_delete or self.can_approve
@property
def allow_link(self):
return self.approved or not (self.can_redact and self.can_approve)
@property
def is_pdf(self):
return self.filetype in PDF_FILETYPES or (
self.name.endswith('.pdf')
and self.filetype == 'application/octet-stream')
@property
def is_image(self):
return (self.filetype.startswith('image/')
or self.filetype in IMAGE_FILETYPES or self.name.endswith(
('.jpg', '.jpeg', '.gif', '.png')))
@property
def is_mail_decoration(self):
return self.is_image and self.size and self.size < 1024 * 60
@property
def is_irrelevant(self):
return self.is_mail_decoration or self.is_signature
@property
def is_signature(self):
return self.name.endswith(
('.p7s', '.vcf', '.asc')) and self.size < 1024 * 15
@property
def can_embed(self):
return self.filetype in EMBEDDABLE_FILETYPES or self.is_pdf
def get_anchor_url(self):
if self.belongs_to:
return '%s#%s' % (self.belongs_to.request.get_absolute_url(),
self.get_html_id())
return '#' + self.get_html_id()
def get_domain_anchor_url(self):
return '%s%s' % (settings.SITE_URL, self.get_anchor_url())
def get_absolute_url(self):
fr = self.belongs_to.request
return reverse('foirequest-show_attachment',
kwargs={
'slug': fr.slug,
'message_id': self.belongs_to.pk,
'attachment_name': self.name
})
def get_absolute_domain_url(self):
return '%s%s' % (settings.SITE_URL, self.get_absolute_url())
def get_absolute_file_url(self, authenticated=False):
if not self.name:
return ''
url = reverse('foirequest-auth_message_attachment',
kwargs={
'message_id': self.belongs_to_id,
'attachment_name': self.name
})
if settings.FOI_MEDIA_TOKENS and authenticated:
signer = TimestampSigner()
value = signer.sign(url).split(':', 1)[1]
return '%s?token=%s' % (url, value)
return url
def get_file_url(self):
return self.get_absolute_domain_file_url()
def get_file_path(self):
if self.file:
return self.file.path
return ''
def get_authenticated_absolute_domain_file_url(self):
return self.get_absolute_domain_file_url(authenticated=True)
def get_absolute_domain_file_url(self, authenticated=False):
return '%s%s' % (settings.FOI_MEDIA_DOMAIN,
self.get_absolute_file_url(
authenticated=authenticated))
def check_token(self, request):
token = request.GET.get('token')
if token is None:
return None
original = '%s:%s' % (self.get_absolute_file_url(), token)
signer = TimestampSigner()
try:
signer.unsign(original, max_age=settings.FOI_MEDIA_TOKEN_EXPIRY)
except SignatureExpired:
return None
except BadSignature:
return False
return True
def approve_and_save(self):
self.approved = True
self.save()
self.attachment_published.send(sender=self)
def remove_file_and_delete(self):
if self.file:
other_references = FoiAttachment.objects.filter(
file=self.file.name).exclude(id=self.id).exists()
if not other_references:
self.file.delete(save=False)
self.delete()
def can_convert_to_pdf(self):
ft = self.filetype.lower()
name = self.name.lower()
return (self.converted_id is None
and can_convert_to_pdf(ft, name=name))
def create_document(self, title=None):
if self.document is not None:
return self.document
if not self.is_pdf:
return
foirequest = self.belongs_to.request
doc = Document.objects.create(
original=self,
user=foirequest.user,
public=foirequest.public,
title=title or self.name,
foirequest=self.belongs_to.request,
publicbody=self.belongs_to.sender_public_body)
self.document = doc
self.save()
return doc
class Command(BaseCommand):
help = "Moves files to content hash based directory structure"
def add_arguments(self, parser):
parser.add_argument('directory', type=str)
parser.add_argument('run_all', nargs='?', type=str)
parser.add_argument('run_now', nargs='?', type=str)
def handle(self, *args, **options):
translation.activate(settings.LANGUAGE_CODE)
if not hasattr(os, 'scandir'):
raise NotImplementedError('Requires Python 3.5+')
self.storage = HashedFilenameStorage()
self.directory = options['directory']
self.run_all = options.get('run_all') == 'all'
self.run_now = options.get('run_now') == 'run-now'
self.missing_attachments = []
if not self.run_now:
print('DRY RUN MODE')
else:
print('Running for real now')
for folder in os.scandir(self.directory):
if folder.is_dir():
self.handle_folder(folder)
with open('missing_attachments.json', 'w') as f:
json.dump(self.missing_attachments, f)
def handle_folder(self, folder):
for file_entry in os.scandir(folder.path):
if file_entry.is_file():
message_id = int(folder.name)
all_attachments = FoiAttachment.objects.filter(
belongs_to_id=message_id
)
result = self.handle_file(file_entry, all_attachments)
if self.run_now and not self.run_all and result:
raise Exception('Not running all')
print('-' * 20)
def handle_file(self, file_entry, all_attachments):
orig_file_path = os.path.abspath(file_entry.path)
print(orig_file_path)
attachment = self.get_attachment(file_entry.name, all_attachments)
if attachment is None:
print('Missing attachments')
self.missing_attachments.append(orig_file_path)
return False
print(attachment.name, attachment.file.name)
file_path = self.fix_filepath(orig_file_path)
fixed_attachment_name = os.path.basename(file_path)
fixed_attachment_name = self.get_unique_name(
fixed_attachment_name, all_attachments
)
attachment.name = fixed_attachment_name
upload_path = upload_to(attachment, fixed_attachment_name)
with open(orig_file_path, 'rb') as f:
stored_path = self.storage._get_content_name(upload_path, f)
print(fixed_attachment_name, stored_path)
new_path = os.path.join(settings.MEDIA_ROOT, stored_path)
print('Move from %s to %s' % (orig_file_path, new_path))
new_base_path = os.path.dirname(new_path)
if not os.path.exists(new_path):
print('Creating dirs for', new_base_path)
if self.run_now:
mode_kwarg = {}
if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
mode_kwarg['mode'] = settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS
os.makedirs(
new_base_path,
exist_ok=True,
**mode_kwarg
)
shutil.move(orig_file_path, new_path)
else:
print('File already exists!', new_base_path)
if self.run_now:
attachment.file.name = stored_path
attachment.save()
return True
def fix_filepath(self, filename):
match = BROKEN_ISO_NAME.match(filename)
if match is not None:
return BROKEN_ISO_NAME.sub('\\1\\2', filename)
return filename
def get_attachment(self, name, attachments):
for att in attachments:
if att.file.name.endswith(name):
return att
def get_unique_name(self, name, attachments):
name_counter = Counter([a.name for a in attachments])
index = 0
while name_counter[name] > 1:
index += 1
path, ext = os.path.splitext(name)
name = '%s_%d%s' % (path, index, ext)
return name
class Command(BaseCommand):
help = "Moves files to content hash based directory structure"
def add_arguments(self, parser):
parser.add_argument('directory', type=str)
parser.add_argument('run_all', nargs='?', type=str)
parser.add_argument('run_now', nargs='?', type=str)
def handle(self, *args, **options):
translation.activate(settings.LANGUAGE_CODE)
if not hasattr(os, 'scandir'):
raise NotImplemented('Requires Python 3.5+')
self.storage = HashedFilenameStorage()
self.directory = options['directory']
self.run_all = options.get('run_all') == 'all'
self.run_now = options.get('run_now') == 'run-now'
self.missing_attachments = []
if not self.run_now:
print('DRY RUN MODE')
else:
print('Running for real now')
for folder in os.scandir(self.directory):
if folder.is_dir():
self.handle_folder(folder)
with open('missing_attachments.json', 'w') as f:
json.dump(self.missing_attachments, f)
def handle_folder(self, folder):
for file_entry in os.scandir(folder.path):
if file_entry.is_file():
message_id = int(folder.name)
all_attachments = FoiAttachment.objects.filter(
belongs_to_id=message_id)
result = self.handle_file(file_entry, all_attachments)
if self.run_now and not self.run_all and result:
raise Exception('Not running all')
print('-' * 20)
def handle_file(self, file_entry, all_attachments):
orig_file_path = os.path.abspath(file_entry.path)
print(orig_file_path)
attachment = self.get_attachment(file_entry.name, all_attachments)
if attachment is None:
print('Missing attachments')
self.missing_attachments.append(orig_file_path)
return False
print(attachment.name, attachment.file.name)
file_path = self.fix_filepath(orig_file_path)
fixed_attachment_name = os.path.basename(file_path)
fixed_attachment_name = self.get_unique_name(fixed_attachment_name,
all_attachments)
attachment.name = fixed_attachment_name
upload_path = upload_to(attachment, fixed_attachment_name)
with open(orig_file_path, 'rb') as f:
stored_path = self.storage._get_content_name(upload_path, f)
print(fixed_attachment_name, stored_path)
new_path = os.path.join(settings.MEDIA_ROOT, stored_path)
print('Move from %s to %s' % (orig_file_path, new_path))
new_base_path = os.path.dirname(new_path)
if not os.path.exists(new_path):
print('Creating dirs for', new_base_path)
if self.run_now:
mode_kwarg = {}
if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
mode_kwarg[
'mode'] = settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS
os.makedirs(new_base_path, exist_ok=True, **mode_kwarg)
shutil.move(orig_file_path, new_path)
else:
print('File already exists!', new_base_path)
if self.run_now:
attachment.file.name = stored_path
attachment.save()
return True
def fix_filepath(self, filename):
match = BROKEN_ISO_NAME.match(filename)
if match is not None:
return BROKEN_ISO_NAME.sub('\\1\\2', filename)
return filename
def get_attachment(self, name, attachments):
for att in attachments:
if att.file.name.endswith(name):
return att
def get_unique_name(self, name, attachments):
name_counter = Counter([a.name for a in attachments])
index = 0
while name_counter[name] > 1:
index += 1
path, ext = os.path.splitext(name)
name = '%s_%d%s' % (path, index, ext)
return name