def _get_error_for_field(self, key, value): # noqa: C901
if self.is_address_column(key):
return
if Columns.make_key(
key) in self.recipient_column_headers_as_column_keys:
if value in [None, ''] or isinstance(value, list):
if self.duplicate_recipient_column_headers:
return None
else:
return Cell.missing_field_error
try:
validate_recipient(
value,
self.template_type,
allow_international_sms=self.allow_international_sms)
except (InvalidEmailError, InvalidPhoneError) as error:
return str(error)
if Columns.make_key(key) not in self.placeholders_as_column_keys:
return
if value in [None, '']:
return Cell.missing_field_error
def get_placeholder_form_instance(
placeholder_name,
dict_to_populate_from,
template_type,
optional_placeholder=False,
allow_international_phone_numbers=False,
):
if (
Columns.make_key(placeholder_name) == 'emailaddress'
and template_type == 'email'
):
field = email_address(label=placeholder_name, gov_user=False)
elif (
Columns.make_key(placeholder_name) == 'phonenumber'
and template_type == 'sms'
):
if allow_international_phone_numbers:
field = international_phone_number(label=placeholder_name)
else:
field = uk_mobile_number(label=placeholder_name)
elif optional_placeholder:
field = StringField(placeholder_name)
else:
field = StringField(placeholder_name, validators=[
DataRequired(message='Can’t be empty')
])
PlaceholderForm.placeholder_value = field
return PlaceholderForm(
placeholder_value=dict_to_populate_from.get(placeholder_name, '')
)
def validate_address(address_line, column):
if Columns.make_key(column) in Columns.from_keys(optional_address_columns).keys():
return address_line
if Columns.make_key(column) not in Columns.from_keys(first_column_headings['letter']).keys():
raise TypeError
if not address_line or not strip_whitespace(address_line):
raise InvalidAddressError('Missing')
return address_line
def values(self, value):
if not value:
self._values = {}
else:
placeholders = Columns.from_keys(self.placeholders)
self._values = Columns(value).as_dict_with_keys(
self.placeholders
| set(key for key in value.keys()
if Columns.make_key(key) not in placeholders.keys()))
def values_with_default_optional_address_lines(self):
keys = Columns.from_keys(
set(self.values.keys()) | {
'address line 3',
'address line 4',
'address line 5',
'address line 6',
}).keys()
return {key: Columns(self.values).get(key) or '' for key in keys}
def duplicate_recipient_column_headers(self):
raw_recipient_column_headers = [
Columns.make_key(column_header)
for column_header in self._raw_column_headers if Columns.make_key(
column_header) in self.recipient_column_headers_as_column_keys
]
return OrderedSet((column_header
for column_header in self._raw_column_headers
if raw_recipient_column_headers.count(
Columns.make_key(column_header)) > 1))
def placeholders(self, value):
try:
self._placeholders = list(value) + self.recipient_column_headers
except TypeError:
self._placeholders = self.recipient_column_headers
self.placeholders_as_column_keys = [
Columns.make_key(placeholder) for placeholder in self._placeholders
]
self.recipient_column_headers_as_column_keys = [
Columns.make_key(placeholder)
for placeholder in self.recipient_column_headers
]
def missing_column_headers(self):
required = {
Columns.make_key(key): key
for key in set([self.recipient_column_header] + self.placeholders)
}
return set(
required[key] for key in set(
[Columns.make_key(self.recipient_column_header)] + self.placeholders_as_column_keys
) - set(
Columns.make_key(column_header) for column_header in self.column_headers
)
)
def test_missing_data():
partial_row = partial(
Row,
row_dict={},
index=1,
error_fn=None,
recipient_column_headers=[],
placeholders=[],
template=None,
)
assert Columns({})['foo'] is None
assert Columns({}).get('foo') is None
assert Columns({}).get('foo', 'bar') == 'bar'
assert partial_row()['foo'] == Cell()
assert partial_row().get('foo') == Cell()
assert partial_row().get('foo', 'bar') == 'bar'
def get_back_link(service_id, template, step_index, placeholders=None):
if step_index == 0:
if should_skip_template_page(template.template_type):
return url_for(
'.choose_template',
service_id=service_id,
)
else:
return url_for(
'.view_template',
service_id=service_id,
template_id=template.id,
)
if template.template_type == 'letter' and placeholders:
# Make sure we’re not redirecting users to a page which will
# just redirect them forwards again
back_link_destination_step_index = next(
(index for index, placeholder in reversed(
list(enumerate(placeholders[:step_index]))) if placeholder
not in Columns(PostalAddress('').as_personalisation)), 1)
return get_back_link(service_id, template,
back_link_destination_step_index + 1)
return url_for(
'main.send_one_off_step',
service_id=service_id,
template_id=template.id,
step_index=step_index - 1,
)
def rows(self):
column_headers = self._raw_column_headers # this is for caching
length_of_column_headers = len(column_headers)
rows_as_lists_of_columns = self._rows
next(rows_as_lists_of_columns) # skip the header row
for row in rows_as_lists_of_columns:
output_dict = OrderedDict()
for column_name, column_value in zip(column_headers, row):
insert_or_append_to_dict(output_dict, column_name, column_value
or None)
length_of_row = len(row)
if length_of_column_headers < length_of_row:
output_dict[None] = row[length_of_column_headers:]
elif length_of_column_headers > length_of_row:
for key in column_headers[length_of_row:]:
insert_or_append_to_dict(output_dict, key, None)
yield Columns(output_dict)
def rows_with_missing_data(self):
return set(
row['index'] for row in self.annotated_rows if any(
str(key) not in Columns.make_key(self.recipient_column_header) and value.get('error')
for key, value in row['columns'].items()
)
)
def save_letter(
self,
service_id,
notification_id,
encrypted_notification,
):
notification = encryption.decrypt(encrypted_notification)
postal_address = PostalAddress.from_personalisation(
Columns(notification['personalisation']))
service = SerialisedService.from_id(service_id)
template = SerialisedTemplate.from_id_and_service_id(
notification['template'],
service_id=service.id,
version=notification['template_version'],
)
try:
# if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
status = NOTIFICATION_CREATED if not service.research_mode else NOTIFICATION_SENDING
saved_notification = persist_notification(
template_id=notification['template'],
template_version=notification['template_version'],
postage=postal_address.postage
if postal_address.international else template.postage,
recipient=postal_address.normalised,
service=service,
personalisation=notification['personalisation'],
notification_type=LETTER_TYPE,
api_key_id=None,
key_type=KEY_TYPE_NORMAL,
created_at=datetime.utcnow(),
job_id=notification['job'],
job_row_number=notification['row_number'],
notification_id=notification_id,
reference=create_random_identifier(),
reply_to_text=template.reply_to_text,
status=status)
if not service.research_mode:
letters_pdf_tasks.get_pdf_for_templated_letter.apply_async(
[str(saved_notification.id)],
queue=QueueNames.CREATE_LETTERS_PDF)
elif current_app.config['NOTIFY_ENVIRONMENT'] in [
'preview', 'development'
]:
research_mode_tasks.create_fake_letter_response_file.apply_async(
(saved_notification.reference, ),
queue=QueueNames.RESEARCH_MODE)
else:
update_notification_status_by_reference(
saved_notification.reference, 'delivered')
current_app.logger.debug("Letter {} created at {}".format(
saved_notification.id, saved_notification.created_at))
except SQLAlchemyError as e:
handle_exception(self, notification, notification_id, e)
def placeholders(self, value):
try:
self._placeholders = list(value)
self.placeholders_as_column_keys = [
Columns.make_key(placeholder) for placeholder in value
]
except TypeError:
self._placeholders, self.placeholders_as_column_keys = [], []
def test_columns_as_dict_with_keys():
assert Columns({
'Date of Birth': '01/01/2001',
'TOWN': 'London'
}).as_dict_with_keys({'date_of_birth', 'town'}) == {
'date_of_birth': '01/01/2001',
'town': 'London'
}
def missing_column_headers(self):
return set(
key for key in self.placeholders
if (
Columns.make_key(key) not in self.column_headers_as_column_keys and
not self.is_optional_address_column(key)
)
)
def test_columns_as_dict():
assert dict(Columns({
'date of birth': '01/01/2001',
'TOWN': 'London'
})) == {
'dateofbirth': '01/01/2001',
'town': 'London'
}
def test_missing_data():
partial_row = partial(
Row,
row_dict={},
index=1,
error_fn=None,
recipient_column_headers=[],
placeholders=[],
template=None,
allow_international_letters=False,
)
with pytest.raises(KeyError):
Columns({})['foo']
assert Columns({}).get('foo') is None
assert Columns({}).get('foo', 'bar') == 'bar'
assert partial_row()['foo'] == Cell()
assert partial_row().get('foo') == Cell()
assert partial_row().get('foo', 'bar') == 'bar'
def get_spreadsheet_column_headings_from_template(template):
column_headings = []
for column_heading in (first_column_headings[template.template_type] +
list(template.placeholders)):
if column_heading not in Columns.from_keys(column_headings):
column_headings.append(column_heading)
return column_headings
def __str__(self):
return Markup(
self.jinja_template.render({
'admin_base_url':
self.admin_base_url,
'logo_file_name':
self.logo_file_name,
'subject':
self.subject,
'message':
Take(
Field(
strip_dvla_markup(self.content),
self.values,
html='escape',
markdown_lists=True,
redact_missing_personalisation=self.
redact_missing_personalisation,
)).then(strip_pipes).then(
make_markdown_take_notice_of_multiple_newlines).then(
notify_letter_preview_markdown).
then(strip_characters_inserted_to_force_newlines).then(
do_nice_typography).then(remove_trailing_linebreak).then(
replace_hyphens_with_non_breaking_hyphens).then(
tweak_dvla_list_markup),
'address':
Take(
Field(self.address_block,
(self.values_with_default_optional_address_lines
if all(
Columns(self.values).get(key) for key in {
'address line 1',
'address line 2',
'postcode',
}) else self.values),
html='escape',
with_brackets=False)
).then(strip_pipes).then(remove_empty_lines).then(
remove_whitespace_before_punctuation).then(nl2li),
'contact_block':
Take(
Field(
'\n'.join(line.strip()
for line in self.contact_block.split('\n')),
self.values,
redact_missing_personalisation=self.
redact_missing_personalisation,
html='escape',
)).then(remove_whitespace_before_punctuation).then(
nl2br).then(strip_pipes),
'date':
datetime.utcnow().strftime('%-d %B %Y')
}))
def _get_error_for_field(self, key, value):
if key == Columns.make_key(self.recipient_column_header):
try:
validate_recipient(value, self.template_type)
except (InvalidEmailError, InvalidPhoneError) as error:
return str(error)
if key not in self.placeholders_as_column_keys:
return
if value in [None, '']:
return 'Missing'
def _address_block(self):
return Take(
Field(self.address_block,
(self.values_with_default_optional_address_lines if all(
Columns(self.values).get(key) for key in {
'address line 1',
'address line 2',
'postcode',
}) else self.values),
html='escape',
with_brackets=False)).then(strip_pipes).then(
remove_empty_lines).then(
remove_whitespace_before_punctuation).then(nl2li)
def has_recipient_columns(self):
if self.template_type == 'letter':
sets_to_check = [
Columns.from_keys(
address_lines_1_to_6_and_postcode_keys).keys(),
Columns.from_keys(address_lines_1_to_7_keys).keys(),
]
else:
sets_to_check = [
self.recipient_column_headers_as_column_keys,
]
for set_to_check in sets_to_check:
if len(
# Work out which columns are shared between the possible
# letter address columns and the columns in the user’s
# spreadsheet (`&` means set intersection)
set_to_check & self.column_headers_as_column_keys
) >= self.count_of_required_recipient_columns:
return True
return False
def _get_error_for_field(self, key, value):
if self.is_optional_address_column(key):
return
if Columns.make_key(
key) in self.recipient_column_headers_as_column_keys:
if value in [None, '']:
return Cell.missing_field_error
try:
validate_recipient(value,
self.template_type,
column=key,
international_sms=self.international_sms)
except (InvalidEmailError, InvalidPhoneError,
InvalidAddressError) as error:
return str(error)
if Columns.make_key(key) not in self.placeholders_as_column_keys:
return
if value in [None, '']:
return Cell.missing_field_error
def _address_block(self):
from notifications_utils.postal_address import PostalAddress
postal_address = PostalAddress.from_personalisation(Columns(self.values))
if postal_address.has_enough_lines and not postal_address.has_too_many_lines:
return postal_address.normalised_lines
return Field(
self.address_block,
self.values,
html='escape',
with_brackets=False,
).splitlines()
def get_spreadsheet_column_headings_from_template(template):
column_headings = []
if template.template_type == 'letter':
# We want to avoid showing `address line 7` for now
recipient_columns = letter_address_columns
else:
recipient_columns = first_column_headings[template.template_type]
for column_heading in (recipient_columns + list(template.placeholders)):
if column_heading not in Columns.from_keys(column_headings):
column_headings.append(column_heading)
return column_headings
def get_rows(self):
column_headers = self._raw_column_headers # this is for caching
length_of_column_headers = len(column_headers)
rows_as_lists_of_columns = self._rows
next(rows_as_lists_of_columns, None) # skip the header row
for index, row in enumerate(rows_as_lists_of_columns):
output_dict = OrderedDict()
for column_name, column_value in zip(column_headers, row):
column_value = strip_and_remove_obscure_whitespace(
column_value)
if Columns.make_key(
column_name
) in self.recipient_column_headers_as_column_keys:
output_dict[column_name] = column_value or None
else:
insert_or_append_to_dict(output_dict, column_name,
column_value or None)
length_of_row = len(row)
if length_of_column_headers < length_of_row:
output_dict[None] = row[length_of_column_headers:]
elif length_of_column_headers > length_of_row:
for key in column_headers[length_of_row:]:
insert_or_append_to_dict(output_dict, key, None)
if index < self.max_rows:
yield Row(
output_dict,
index=index,
error_fn=self._get_error_for_field,
recipient_column_headers=self.recipient_column_headers,
placeholders=self.placeholders_as_column_keys,
template=self.template,
allow_international_letters=self.
allow_international_letters,
)
else:
yield None
def get_example_csv_rows(template,
use_example_as_example=True,
submitted_fields=False):
return {
'email': ['*****@*****.**']
if use_example_as_example else [current_user.email_address],
'sms': ['6502532222']
if use_example_as_example else [current_user.mobile_number],
'letter': [(submitted_fields or {}).get(
key,
get_example_letter_address(key) if use_example_as_example else key)
for key in first_column_headings['letter']]
}[template.template_type] + get_example_csv_fields(
(placeholder
for placeholder in template.placeholders if placeholder not in
Columns.from_keys(first_column_headings[template.template_type])),
use_example_as_example, submitted_fields)
def get_annotated_rows(self):
if self.too_many_rows:
return []
for row_index, row in enumerate(self.rows):
if self.template:
self.template.values = dict(row.items())
yield dict(
columns=Columns({key: {
'data': value,
'error': self._get_error_for_field(key, value),
'ignore': (
key != Columns.make_key(self.recipient_column_header) and
key not in self.placeholders_as_column_keys
)
} for key, value in row.items()}),
index=row_index,
message_too_long=bool(self.template and self.template.content_too_long)
)
def get_annotated_rows(self):
if self.too_many_rows:
return []
for row_index, row in enumerate(self.rows):
if self.template:
self.template.values = dict(row.items())
yield dict(
columns=Columns({
key: {
'data': value,
'error': self._get_error_for_field(key, value),
'ignore': (key not in self.placeholders_as_column_keys)
}
for key, value in row.items()
}),
index=row_index,
message_too_long=bool(
self.template
and self.template.content_count > self.character_limit))
def send_notification(service_id, template_id):
if {'recipient', 'placeholders'} - set(session.keys()):
return redirect(
url_for(
'.send_one_off',
service_id=service_id,
template_id=template_id,
))
db_template = current_service.get_template_with_user_permission_or_403(
template_id, current_user)
try:
noti = notification_api_client.send_notification(
service_id,
template_id=db_template['id'],
recipient=session['recipient']
or Columns(session['placeholders'])['address line 1'],
personalisation=session['placeholders'],
sender_id=session['sender_id'] if 'sender_id' in session else None)
except HTTPError as exception:
current_app.logger.info(
'Service {} could not send notification: "{}"'.format(
current_service.id, exception.message))
return render_template(
'views/notifications/check.html',
**_check_notification(service_id, template_id, exception),
)
session.pop('placeholders')
session.pop('recipient')
session.pop('sender_id', None)
return redirect(
url_for(
'.view_notification',
service_id=service_id,
notification_id=noti['id'],
# used to show the final step of the tour (help=3) or not show
# a back link on a just sent one off notification (help=0)
help=request.args.get('help')))
def __str__(self):
return Markup(
self.jinja_template.render({
'admin_base_url':
self.admin_base_url,
'logo_file_name':
self.logo_file_name,
'subject':
self.subject,
'message':
Take.as_field(
strip_dvla_markup(self.content),
self.values,
html='escape',
markdown_lists=True).then(strip_pipes).then(
prepare_newlines_for_markdown).then(
notify_letter_preview_markdown).as_string,
'address':
Take.as_field(
self.address_block,
(self.values_with_default_optional_address_lines if all(
Columns(self.values).get(key) for key in {
'address line 1',
'address line 2',
'postcode',
}) else self.values),
html='escape',
with_brackets=False).then(strip_pipes).then(
remove_empty_lines).then(nl2br).as_string,
'contact_block':
Take.as_field(
'\n'.join(line.strip()
for line in self.contact_block.split('\n')),
self.values,
html='escape',
).then(nl2br).then(strip_pipes).as_string,
'date':
datetime.utcnow().strftime('%-d %B %Y')
}))
def __init__(self, old_template, new_template):
self.old_placeholders = Columns.from_keys(old_template.placeholders)
self.new_placeholders = Columns.from_keys(new_template.placeholders)
def has_recipient_column(self):
return Columns.make_key(self.recipient_column_header) in set(
Columns.make_key(column_header) for column_header in self.column_headers
)
def test_lookup(key, should_be_present, in_dictionary):
assert (key in Columns(in_dictionary)) == should_be_present
def has_recipient_columns(self):
return set(
Columns.make_key(recipient_column)
for recipient_column in self.recipient_column_headers
if not self.is_optional_address_column(recipient_column)
) <= self.column_headers_as_column_keys
def is_optional_address_column(self, key):
return (self.template_type == 'letter' and Columns.make_key(key)
in Columns.from_keys(optional_address_columns).keys())