def test_created_document_is_indexed(self, index_mock):
form = DemoMetadataForm({
'title': 'Title',
}, category=self.category)
rev_form = DemoMetadataRevisionForm({
'docclass': '1',
'received_date': '2015-01-01',
'created_on': '2015-01-01',
}, category=self.category)
save_document_forms(form, rev_form, self.category)
self.assertEqual(index_mock.call_count, 1)
def test_created_document_is_indexed(self, index_mock):
form = DemoMetadataForm({
'title': 'Title',
}, category=self.category)
rev_form = DemoMetadataRevisionForm(
{
'docclass': '1',
'received_date': '2015-01-01',
'created_on': '2015-01-01',
},
category=self.category)
save_document_forms(form, rev_form, self.category)
self.assertEqual(index_mock.call_count, 1)
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
doc, metadata, revision = save_document_forms(
document_form,
revision_form,
self.category,
created_by=self.request.user)
message_text = '''You created the document
<a href="%(url)s">%(key)s (%(title)s)</a>'''
message_data = {
'url': doc.get_absolute_url(),
'key': doc.document_key,
'title': doc.title
}
notify(self.request.user, _(message_text) % message_data)
activity_log.send(
verb='created',
target=None,
action_object=doc,
sender=None,
actor=self.request.user)
return HttpResponseRedirect(self.get_success_url())
def save_to_document(self):
"""Use self data to create / update the corresponding revision."""
fields, files = self.get_document_fields()
kwargs = {
'category': self.category,
'data': fields,
'files': files
}
# The document was created earlier during
# the batch import
if self.document is None and self.revision > 0:
self.document = Document.objects.get(document_key=self.document_key)
metadata = getattr(self.document, 'metadata', None)
kwargs.update({'instance': metadata})
Form = self.category.get_metadata_form_class()
metadata_form = Form(**kwargs)
# If there is no such revision, the method will return None
# which is fine.
revision = metadata.get_revision(self.revision) if metadata else None
kwargs.update({'instance': revision})
RevisionForm = self.category.get_revision_form_class()
revision_form = RevisionForm(**kwargs)
doc, meta, rev = save_document_forms(
metadata_form, revision_form, self.category)
# Performs custom import action
rev.post_trs_import(self)
def save_to_document(self):
"""Use self data to create / update the corresponding revision."""
fields, files = self.get_document_fields()
kwargs = {
'category': self.category,
'data': fields,
'files': files
}
# The document was created earlier during
# the batch import
if self.document is None and self.revision > 0:
self.document = Document.objects.get(document_key=self.document_key)
metadata = getattr(self.document, 'metadata', None)
kwargs.update({'instance': metadata})
Form = self.category.get_metadata_form_class()
metadata_form = Form(**kwargs)
# If there is no such revision, the method will return None
# which is fine.
revision = metadata.get_revision(self.revision) if metadata else None
kwargs.update({'instance': revision})
RevisionForm = self.category.get_revision_form_class()
revision_form = RevisionForm(**kwargs)
doc, meta, rev = save_document_forms(
metadata_form, revision_form, self.category)
# Performs custom import action
rev.post_trs_import(self)
def do_import(self, line):
assert hasattr(self, 'data')
self.line = line
# Checking if the document already exists
key = self.data.get('document_key', None)
doc = get_object_or_None(Document, document_key=key)
metadata = doc.metadata if doc else None
# Checking if the revision already exists
revision_num = self.data.get('revision', None)
revision = metadata.get_revision(revision_num) if metadata and revision_num else None
form, revision_form = self.get_forms(metadata, revision)
try:
if form.is_valid() and revision_form.is_valid():
doc, metadata, revision = save_document_forms(
form, revision_form, self.batch.category)
self.document = doc
self.status = self.STATUSES.success
else:
errors = dict(form.errors.items() + revision_form.errors.items())
self.errors = json.dumps(errors)
self.status = self.STATUSES.error
except Exception as e:
self.errors = json.dumps({
'An error occurred': [str(e)]
})
self.status = self.STATUSES.error
def create_transmittal(from_category, to_category, revisions, contract_nb,
recipient, **form_data):
"""Create an outgoing transmittal with the given revisions."""
# Do we have a list of revisions?
if not isinstance(revisions, list) or len(revisions) == 0:
raise errors.MissingRevisionsError(
'Please provide a valid list of transmittals')
# The "from" category must contain transmittable documents
from_type = from_category.revision_class()
if not issubclass(from_type, TransmittableMixin):
raise errors.InvalidCategoryError(
'Source category must contain transmittable documents')
# The "destination" category must contain transmittals
dest_type = to_category.revision_class()
if not issubclass(dest_type, OutgoingTransmittalRevision):
raise errors.InvalidCategoryError(
'Destination category must contain transmittals')
# The recipient must be linked to the "from" category
if from_category not in recipient.linked_categories.all():
raise errors.InvalidRecipientError(
'Recipient is not linked to the document category')
# Do we have valid revisions?
for rev in revisions:
if not isinstance(rev, TransmittableMixin):
raise errors.InvalidRevisionsError(
'At least one of the revisions is invalid.')
if not rev.can_be_transmitted:
raise errors.InvalidRevisionsError(
'At least one of the rivisions cannot be transmitted')
if not rev.document.category == from_category:
raise errors.InvalidRevisionsError(
'Some revisions are not from the correct category')
originator = from_category.organisation.trigram
sequential_number = find_next_trs_number(originator, recipient, contract_nb)
form_data.update({
'contract_number': contract_nb,
'originator': originator,
'recipient': recipient.id,
'sequential_number': sequential_number,
'created_on': timezone.now(),
'received_date': timezone.now(),
})
# Let's create the transmittal, then
trs_form = OutgoingTransmittalForm(form_data, category=to_category)
revision_form = OutgoingTransmittalRevisionForm(form_data, category=to_category)
with transaction.atomic():
doc, trs, revision = save_document_forms(trs_form, revision_form, to_category)
trs.link_to_revisions(revisions)
return doc, trs, revision
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
doc, metadata, revision = save_document_forms(document_form, revision_form, self.category)
message_text = """You created the document
<a href="%(url)s">%(key)s (%(title)s)</a>"""
message_data = {"url": doc.get_absolute_url(), "key": doc.document_key, "title": doc.title}
notify(self.request.user, _(message_text) % message_data)
return HttpResponseRedirect(self.get_success_url())
def test_revised_document_is_indexed(self, index_mock):
form = DemoMetadataForm({
'title': 'Title',
}, category=self.category)
rev_form = DemoMetadataRevisionForm({
'docclass': '1',
'received_date': '2015-01-01',
'created_on': '2015-01-01',
}, category=self.category)
doc, meta, rev = save_document_forms(form, rev_form, self.category)
revision = doc.latest_revision
revision.pk = None
revision.save()
doc.save()
self.assertEqual(index_mock.call_count, 2)
def test_revised_document_is_indexed(self, index_mock):
form = DemoMetadataForm({
'title': 'Title',
}, category=self.category)
rev_form = DemoMetadataRevisionForm(
{
'docclass': '1',
'received_date': '2015-01-01',
'created_on': '2015-01-01',
},
category=self.category)
doc, meta, rev = save_document_forms(form, rev_form, self.category)
revision = doc.latest_revision
revision.pk = None
revision.save()
doc.save()
self.assertEqual(index_mock.call_count, 2)
def do_import(self, line):
assert hasattr(self, 'data')
self.line = line
# Checking if the document already exists
key = self.data.get('document_key', None)
doc = get_object_or_None(Document, document_key=key)
metadata = doc.metadata if doc else None
# Processing csv data to denormalize foreign keys
self.denormalize_data(self.batch.category)
# In case of denormalization error, we exit
if self.status == self.STATUSES.error:
return
# Checking if the revision already exists
revision_num = self.data.get('revision', None)
revision = metadata.get_revision(
revision_num) if metadata and revision_num else None
form, revision_form = self.get_forms(metadata, revision)
try:
if form.is_valid() and revision_form.is_valid():
# The `save_document_forms` function sends a signal triggering
# ES indexing and schedule field rewriting. Setting
# `rewrite_schedule` to False disables rewriting
# (ES indexing is still enabled)
doc, metadata, revision = save_document_forms(
form,
revision_form,
self.batch.category,
rewrite_schedule=False)
self.document = doc
self.status = self.STATUSES.success
else:
errors = dict(
list(form.errors.items()) +
list(revision_form.errors.items()))
self.errors = json.dumps(errors)
self.status = self.STATUSES.error
except Exception as e:
self.errors = json.dumps({'An error occurred': [str(e)]})
self.status = self.STATUSES.error
def do_import(self, line):
assert hasattr(self, 'data')
self.line = line
# Checking if the document already exists
key = self.data.get('document_key', None)
doc = get_object_or_None(Document, document_key=key)
metadata = doc.metadata if doc else None
# Processing csv data to denormalize foreign keys
self.denormalize_data(self.batch.category)
# In case of denormalization error, we exit
if self.status == self.STATUSES.error:
return
# Checking if the revision already exists
revision_num = self.data.get('revision', None)
revision = metadata.get_revision(revision_num) if metadata and revision_num else None
form, revision_form = self.get_forms(metadata, revision)
try:
if form.is_valid() and revision_form.is_valid():
# The `save_document_forms` function sends a signal triggering
# ES indexing and schedule field rewriting. Setting
# `rewrite_schedule` to False disables rewriting
# (ES indexing is still enabled)
doc, metadata, revision = save_document_forms(
form, revision_form,
self.batch.category,
rewrite_schedule=False)
self.document = doc
self.status = self.STATUSES.success
else:
errors = dict(form.errors.items() + revision_form.errors.items())
self.errors = json.dumps(errors)
self.status = self.STATUSES.error
except Exception as e:
self.errors = json.dumps({
'An error occurred': [str(e)]
})
self.status = self.STATUSES.error
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
document, self.object, self.revision = save_document_forms(
document_form, revision_form, self.category)
message_text = '''You created revision %(rev)s for document
<a href="%(url)s">%(key)s (%(title)s)</a>'''
message_data = {
'rev': self.revision.name,
'url': self.object.get_absolute_url(),
'key': self.object.document_key,
'title': self.object.title
}
notify(self.request.user, _(message_text) % message_data)
activity_log.send(verb=Activity.VERB_CREATED,
target=self.revision,
sender=None,
actor=self.request.user)
return HttpResponseRedirect(self.get_success_url())
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
document, self.object, self.revision = save_document_forms(
document_form, revision_form, self.category)
message_text = '''You created revision %(rev)s for document
<a href="%(url)s">%(key)s (%(title)s)</a>'''
message_data = {
'rev': self.revision.name,
'url': self.object.get_absolute_url(),
'key': self.object.document_key,
'title': self.object.title
}
notify(self.request.user, _(message_text) % message_data)
activity_log.send(verb=Activity.VERB_CREATED,
target=self.revision,
sender=None,
actor=self.request.user)
return HttpResponseRedirect(self.get_success_url())
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
doc, metadata, revision = save_document_forms(document_form,
revision_form,
self.category)
message_text = '''You created the document
<a href="%(url)s">%(key)s (%(title)s)</a>'''
message_data = {
'url': doc.get_absolute_url(),
'key': doc.document_key,
'title': doc.title
}
notify(self.request.user, _(message_text) % message_data)
activity_log.send(verb='created',
target=None,
action_object=doc,
sender=None,
actor=self.request.user)
return HttpResponseRedirect(self.get_success_url())
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
document, self.object, self.revision = save_document_forms(
document_form, revision_form, self.category)
return HttpResponseRedirect(self.get_success_url())
def create_transmittal(from_category, to_category, revisions, contract_nb,
recipient, **form_data):
"""Create an outgoing transmittal with the given revisions."""
# Do we have a list of revisions?
if not isinstance(revisions, list) or len(revisions) == 0:
raise errors.MissingRevisionsError(
'Please provide a valid list of transmittals')
# The "from" category must contain transmittable documents
from_type = from_category.revision_class()
if not issubclass(from_type, TransmittableMixin):
raise errors.InvalidCategoryError(
'Source category must contain transmittable documents')
# The "destination" category must contain transmittals
dest_type = to_category.revision_class()
if not issubclass(dest_type, OutgoingTransmittalRevision):
raise errors.InvalidCategoryError(
'Destination category must contain transmittals')
# The recipient must be linked to the "from" category
if from_category not in recipient.linked_categories.all():
raise errors.InvalidRecipientError(
'Recipient is not linked to the document category')
# The 'contract_nb' must belong to the contracts linked by the category
cat_contracts = from_category.contracts.values_list('number', flat=True)
if contract_nb not in cat_contracts:
raise errors.InvalidContractNumberError(
'Contract number is not linked to the document category')
# Do we have valid revisions?
for rev in revisions:
if not isinstance(rev, TransmittableMixin):
raise errors.InvalidRevisionsError(
'At least one of the revisions is invalid.')
if not rev.can_be_transmitted_to_recipient(recipient=recipient):
raise errors.InvalidRevisionsError(
'At least one of the revisions cannot be transmitted')
if not rev.document.category == from_category:
raise errors.InvalidRevisionsError(
'Some revisions are not from the correct category')
originator = from_category.organisation.trigram
sequential_number = find_next_trs_number(originator, recipient,
contract_nb)
form_data.update({
'sequential_number': sequential_number,
'created_on': timezone.now(),
'received_date': timezone.now(),
})
# Some fields are excluded from the form, so we have to
# provide a base instance instead
base_instance = OutgoingTransmittal(
revisions_category=from_category,
contract_number=contract_nb,
originator=originator,
recipient=recipient,
)
# Let's create the transmittal, then
trs_form = OutgoingTransmittalForm(form_data,
instance=base_instance,
category=to_category)
revision_form = OutgoingTransmittalRevisionForm(form_data,
category=to_category)
with transaction.atomic():
doc, trs, revision = save_document_forms(trs_form, revision_form,
to_category)
trs.link_to_revisions(revisions)
signals.transmittal_created.send(document=doc,
metadata=trs,
revision=revision,
sender=trs.__class__)
return doc, trs, revision
def save(self):
"""Save transmittal data in db."""
# We import those here because we need to make sure that
# the values_lists have already been populated
from transmittals.models import TrsRevision
from transmittals.forms import TransmittalForm, TransmittalRevisionForm
# Build the list of related documents
keys = []
for line in self:
key = line.csv_data['document_key']
if key not in keys:
keys.append(key)
related_documents = Document.objects \
.filter(document_key__in=keys) \
.values_list('id', flat=True)
data = {
'contractor': self.contractor,
'tobechecked_dir': self.tobechecked_dir,
'accepted_dir': self.accepted_dir,
'rejected_dir': self.rejected_dir,
'contract_number': self.contract_number,
'originator': self.originator,
'recipient': self.recipient,
'sequential_number': self.sequential_number,
'status': 'tobechecked',
'related_documents': list(related_documents),
'revision_date': datetime.date.today(),
'received_date': datetime.date.today(),
'created_on': datetime.date.today(),
}
# The csv file is linked in the "native_file" field
native_file = File(open(self.csv_fullname))
files = {
'native_file': native_file,
}
# Use document forms to create the Transmittal
form = TransmittalForm(data=data, category=self.trs_category)
revision_form = TransmittalRevisionForm(data=data,
files=files,
category=self.trs_category)
doc, transmittal, revision = save_document_forms(form,
revision_form,
self.trs_category,
is_indexable=False)
native_file.close()
nb_line = 0
for line in self:
data = line.cleaned_data
metadata = line.get_metadata()
document = getattr(metadata, 'document', None)
if nb_line % 100 == 0:
logger.info('Importing line {} ({})'.format(
nb_line + 1, data['document_key']))
# Is this a revision creation or are we editing an existing one?
if metadata is None:
is_new_revision = True
else:
latest_revision = metadata.latest_revision.revision
is_new_revision = bool(int(data['revision']) > latest_revision)
pdf_file = File(open(line.pdf_fullname, 'rb'))
native_file = line.native_fullname
if native_file:
native_file = File(open(native_file, 'rb'))
data.update({
'transmittal': transmittal,
'document': document,
'is_new_revision': is_new_revision,
'category': self.doc_category,
'pdf_file': pdf_file,
'native_file': native_file,
'sequential_number': line.sequential_number, # XXX Hack
})
TrsRevision.objects.create(**data)
nb_line += 1
pdf_file.close()
if hasattr(native_file, 'close'):
native_file.close()
def create_transmittal(from_category, to_category, revisions, contract_nb,
recipient, **form_data):
"""Create an outgoing transmittal with the given revisions."""
# Do we have a list of revisions?
if not isinstance(revisions, list) or len(revisions) == 0:
raise errors.MissingRevisionsError(
'Please provide a valid list of transmittals')
# The "from" category must contain transmittable documents
from_type = from_category.revision_class()
if not issubclass(from_type, TransmittableMixin):
raise errors.InvalidCategoryError(
'Source category must contain transmittable documents')
# The "destination" category must contain transmittals
dest_type = to_category.revision_class()
if not issubclass(dest_type, OutgoingTransmittalRevision):
raise errors.InvalidCategoryError(
'Destination category must contain transmittals')
# The recipient must be linked to the "from" category
if from_category not in recipient.linked_categories.all():
raise errors.InvalidRecipientError(
'Recipient is not linked to the document category')
# The 'contract_nb' must belong to the contracts linked by the category
cat_contracts = from_category.contracts.values_list('number', flat=True)
if contract_nb not in cat_contracts:
raise errors.InvalidContractNumberError(
'Contract number is not linked to the document category')
# Do we have valid revisions?
for rev in revisions:
if not isinstance(rev, TransmittableMixin):
raise errors.InvalidRevisionsError(
'At least one of the revisions is invalid.')
if not rev.can_be_transmitted_to_recipient(recipient=recipient):
raise errors.InvalidRevisionsError(
'At least one of the revisions cannot be transmitted')
if not rev.document.category == from_category:
raise errors.InvalidRevisionsError(
'Some revisions are not from the correct category')
originator = from_category.organisation.trigram
sequential_number = find_next_trs_number(originator, recipient, contract_nb)
form_data.update({
'sequential_number': sequential_number,
'created_on': timezone.now(),
'received_date': timezone.now(),
})
# Some fields are excluded from the form, so we have to
# provide a base instance instead
base_instance = OutgoingTransmittal(
revisions_category=from_category,
contract_number=contract_nb,
originator=originator,
recipient=recipient,
)
# Let's create the transmittal, then
trs_form = OutgoingTransmittalForm(
form_data, instance=base_instance, category=to_category)
revision_form = OutgoingTransmittalRevisionForm(
form_data, category=to_category)
with transaction.atomic():
doc, trs, revision = save_document_forms(trs_form, revision_form, to_category)
trs.link_to_revisions(revisions)
signals.transmittal_created.send(
document=doc,
metadata=trs,
revision=revision,
sender=trs.__class__)
return doc, trs, revision
def save(self):
"""Save transmittal data in db."""
# We import those here because we need to make sure that
# the values_lists have already been populated
from transmittals.models import TrsRevision
from transmittals.forms import TransmittalForm, TransmittalRevisionForm
# Build the list of related documents
keys = []
for line in self:
key = line.csv_data['document_key']
if key not in keys:
keys.append(key)
related_documents = Document.objects \
.filter(document_key__in=keys) \
.values_list('id', flat=True)
data = {
'contractor': self.contractor,
'tobechecked_dir': self.tobechecked_dir,
'accepted_dir': self.accepted_dir,
'rejected_dir': self.rejected_dir,
'contract_number': self.contract_number,
'originator': self.originator,
'recipient': self.recipient,
'sequential_number': self.sequential_number,
'status': 'tobechecked',
'related_documents': list(related_documents),
'revision_date': datetime.date.today(),
'received_date': datetime.date.today(),
'created_on': datetime.date.today(),
}
# The csv file is linked in the "native_file" field
native_file = File(open(self.csv_fullname))
files = {
'native_file': native_file,
}
# Use document forms to create the Transmittal
form = TransmittalForm(data=data, category=self.trs_category)
revision_form = TransmittalRevisionForm(
data=data, files=files, category=self.trs_category)
doc, transmittal, revision = save_document_forms(
form, revision_form, self.trs_category, is_indexable=False)
native_file.close()
nb_line = 0
for line in self:
data = line.cleaned_data
metadata = line.get_metadata()
document = getattr(metadata, 'document', None)
if nb_line % 100 == 0:
logger.info('Importing line {} ({})'.format(
nb_line + 1, data['document_key']))
# Is this a revision creation or are we editing an existing one?
if metadata is None:
is_new_revision = True
else:
latest_revision = metadata.latest_revision.revision
is_new_revision = bool(int(data['revision']) > latest_revision)
pdf_file = File(open(line.pdf_fullname))
native_file = line.native_fullname
if native_file:
native_file = File(open(native_file))
data.update({
'transmittal': transmittal,
'document': document,
'is_new_revision': is_new_revision,
'category': self.doc_category,
'pdf_file': pdf_file,
'native_file': native_file,
'sequential_number': line.sequential_number, # XXX Hack
})
TrsRevision.objects.create(**data)
nb_line += 1
pdf_file.close()
if hasattr(native_file, 'close'):
native_file.close()
def form_valid(self, document_form, revision_form):
"""Saves both the document and it's revision."""
document, self.object, self.revision = save_document_forms(
document_form, revision_form, self.category
)
return HttpResponseRedirect(self.get_success_url())
