def get_browser_language(request):
"""
Returns *ONLY* the language that is sent by the browser via the
Accept-Language headers. Defaults to ``settings.LANGUAGE_CODE`` if
that doesn't work!
This is the language we switch back to when translation mode is turned off.
Copied from the bottom half of
``django.utils.translation.trans_real.get_language_from_request()``
.. note::
Using ``get_language_from_request()`` doesn't work for us because
it first inspects session and cookies and we've already set Esperanto
in both the session and the cookie!
"""
accept = request.META.get("HTTP_ACCEPT_LANGUAGE", "")
for accept_lang, _unused in trans_real.parse_accept_lang_header(accept):
if accept_lang == "*":
break
if not trans_real.language_code_re.search(accept_lang):
continue
try:
return translation.get_supported_language_variant(accept_lang)
except LookupError:
continue
try:
return translation.get_supported_language_variant(settings.LANGUAGE_CODE)
except LookupError:
return settings.LANGUAGE_CODE
def check_language_settings_consistent(app_configs, **kwargs):
"""Error if language settings are not consistent with each other."""
try:
get_supported_language_variant(settings.LANGUAGE_CODE)
except LookupError:
return [E004]
else:
return []
def get_localized_name(column, language=None):
default_language = translation.get_supported_language_variant(
settings.LANGUAGE_CODE)
if not language:
language = translation.get_supported_language_variant(
translation.get_language())
return column if language in default_language else "%s_%s" % (column,
language)
class ImportTestCase(TestCase):
language = translation.get_supported_language_variant(
translation.get_language())
start_row = config.START_ROW
header_row = config.HEADER_ROW
template = config.DEFAULT_TEMPLATE
date_format = formats.get_format("SHORT_DATE_FORMAT")
params = {
'language': language,
'start_row': start_row,
'header_row': header_row,
'template': template,
'date_format': date_format,
}
def setUp(self):
# creates template
clean_template = Template()
clean_template.id = self.template
clean_template.name = 'Test Template'
clean_template.mapping = json.loads(
'{"contact": {"sex": {"name": "Sexo", "varname": "sex", "required": false}, "country": {"name": "País", "varname": "country", "required": true}, "document": {"name": "Número de Identificación", "varname": "document", "required": false}, "men_home": {"name": "Hombres en su familia", "varname": "men_home", "required": false}, "birthdate": {"name": "Fecha de Nacimiento", "varname": "birthdate", "required": false}, "community": {"name": "Comunidad", "varname": "community", "required": false}, "education": {"name": "Educación", "varname": "education", "required": false}, "last_name": {"name": "Apellidos", "varname": "last_name", "required": false}, "first_name": {"name": "Nombres", "varname": "first_name", "required": true}, "women_home": {"name": "Mujeres en su familia", "varname": "women_home", "required": false}, "municipality": {"name": "Departamento", "varname": "municipality", "required": false}, "organization": {"name": "Organización Perteneciente", "varname": "organization", "required": false}, "phone_personal": {"name": "Teléfono", "varname": "phone_personal", "required": false}}, "project": {"name": {"name": "Nombre de Proyecto", "varname": "name", "required": true}, "organization": {"name": "Organización Implementadora", "varname": "organization", "required": true}}, "project_contact": {"date_entry_project": {"name": "Fecha de ingreso al proyecto", "varname": "date_entry_project", "required": true}}}'
)
clean_template.save()
# creates excel file
excel_name = 'test_excel_file.xlsx'
tmp_excel_name = "{}-{}-{}".format('test',
time.strftime("%Y%m%d-%H%M%S"),
excel_name)
workbook = Workbook()
sheet = workbook.active
sheet.title = 'data'
self.params['excel_file'] = tmp_excel_name
# creates excel content
row = 1
column = 1
sheet.cell(row=row, column=column).value = 'Merged titles'
row += 1
for model in clean_template.mapping:
for field_name, field_details in clean_template.mapping[
model].items():
sheet.cell(row=row,
column=column).value = field_details['name']
column += 1
row += 1
default_storage.save('{}/{}'.format('tmp', tmp_excel_name),
ContentFile(save_virtual_workbook(workbook)))
def test_step3(self):
admin = User.objects.create_superuser('admin',
email=None,
password=None)
c = Client()
c.force_login(admin)
response = c.post('/import/participants/step3', self.params)
self.assertEqual(response.status_code, 200)
def __call__(self, request):
if request.GET.get("lang"):
try:
language_code = translation.get_supported_language_variant(
request.GET["lang"])
except LookupError:
pass
else:
if language_code != request.LANGUAGE_CODE:
translation.activate(language_code)
request.LANGUAGE_CODE = translation.get_language()
return self.get_response(request)
def annotate_with_translated_name(queryset):
language_code = get_language()
language_variant = get_supported_language_variant(language_code)
if language_variant == 'de':
queryset = queryset.annotate(name=Case(
When(~Q(title_de__exact=''), then='title_de'), default='title'))
elif language_variant == 'cs':
queryset = queryset.annotate(name=Case(
When(~Q(title_cs__exact=''), then='title_cs'), default='title'))
else:
queryset = queryset.annotate(name=F('title'))
return queryset
def get_language_from_scope(scope):
"""Get language from ASGI scope.
Based on django.utils.translation.get_language_from_request
"""
accept = ""
for k, v in scope["headers"]:
if k == b"accept-language":
accept = v.decode("latin1")
for accept_lang, unused in parse_accept_lang_header(accept):
if accept_lang == "*":
break
if not language_code_re.search(accept_lang): # pragma: no cover
continue
try:
return get_supported_language_variant(accept_lang)
except LookupError: # pragma: no cover
continue
try:
return get_supported_language_variant(settings.LANGUAGE_CODE)
except LookupError: # pragma: no cover
return settings.LANGUAGE_CODE
def create_initial_language(apps, schema_editor):
Language = apps.get_model('wagtailtrans.Language')
# Only run if there are no languages already (new build)
if not Language.objects.exists():
# The value of LANGUAGE_CODE may not be an entry in LANGUAGES.
# For example, LANGUAGE_CODE may be "en-us" but LANGUAGES may only contain "en".
# This converts the LANGUAGE_CODE into a supported variant if it isn't in LANGUAGES.
language_code = translation.get_supported_language_variant(settings.LANGUAGE_CODE)
Language.objects.create(
code=language_code,
is_default=True,
live=True
)
def get_translation_for_request(self, request):
"""
Returns the translation of this page that should be used to route
the specified request.
"""
language_code = translation.get_supported_language_variant(
request.LANGUAGE_CODE)
try:
locale = Locale.objects.get(language_code=language_code)
return self.get_translation(locale)
except Locale.DoesNotExist:
return
except self.DoesNotExist:
return
def get_content_languages():
"""
Cache of settings.WAGTAIL_CONTENT_LANGUAGES in a dictionary for easy lookups by key.
"""
content_languages = getattr(settings, "WAGTAIL_CONTENT_LANGUAGES", None)
languages = dict(settings.LANGUAGES)
if content_languages is None:
# Default to a single language based on LANGUAGE_CODE
default_language_code = get_supported_language_variant(
settings.LANGUAGE_CODE)
try:
language_name = languages[default_language_code]
except KeyError:
# get_supported_language_variant on the 'null' translation backend (used for
# USE_I18N=False) returns settings.LANGUAGE_CODE unchanged without accounting for
# language variants (en-us versus en), so retry with the generic version.
default_language_code = default_language_code.split("-")[0]
try:
language_name = languages[default_language_code]
except KeyError:
# Can't extract a display name, so fall back on displaying LANGUAGE_CODE instead
language_name = settings.LANGUAGE_CODE
# Also need to tweak the languages dict to get around the check below
languages[default_language_code] = settings.LANGUAGE_CODE
content_languages = [
(default_language_code, language_name),
]
# Check that each content language is in LANGUAGES
for language_code, name in content_languages:
if language_code not in languages:
raise ImproperlyConfigured(
"The language {} is specified in WAGTAIL_CONTENT_LANGUAGES but not LANGUAGES. "
"WAGTAIL_CONTENT_LANGUAGES must be a subset of LANGUAGES.".
format(language_code))
return dict(content_languages)
def get_content_languages():
"""
Cache of settings.WAGTAIL_CONTENT_LANGUAGES in a dictionary for easy lookups by key.
"""
content_languages = getattr(settings, 'WAGTAIL_CONTENT_LANGUAGES', None)
languages = dict(settings.LANGUAGES)
if content_languages is None:
# Default to a single language based on LANGUAGE_CODE
default_language_code = get_supported_language_variant(settings.LANGUAGE_CODE)
content_languages = [
(default_language_code, languages[default_language_code]),
]
# Check that each content language is in LANGUAGES
for language_code, name in content_languages:
if language_code not in languages:
raise ImproperlyConfigured(
"The language {} is specified in WAGTAIL_CONTENT_LANGUAGES but not LANGUAGES. "
"WAGTAIL_CONTENT_LANGUAGES must be a subset of LANGUAGES.".format(language_code)
)
return dict(content_languages)
def process_request(self, request):
if isinstance(request.user, AnonymousUser):
# Anonymous users don't have preferences
language = "auto"
request.theme = settings.DEFAULT_THEME
request.pagesize = settings.DEFAULT_PAGESIZE
else:
if "language" in request.GET:
try:
language = get_supported_language_variant(request.GET["language"])
except Exception:
language = request.user.language
else:
language = request.user.language
request.theme = request.user.theme or settings.DEFAULT_THEME
request.pagesize = request.user.pagesize or settings.DEFAULT_PAGESIZE
if language == "auto":
language = translation.get_language_from_request(request)
if translation.get_language() != language:
translation.activate(language)
request.LANGUAGE_CODE = translation.get_language()
request.charset = settings.DEFAULT_CHARSET
if request.GET.get("navbar", None):
request.session["navbar"] = True
def get(self, request, *args, **kwargs):
"""获取分享目录网页"""
lang_code = request.GET.get(LANGUAGE_QUERY_PARAMETER)
if lang_code:
try:
lang_code = translation.get_supported_language_variant(lang_code)
translation.activate(lang_code)
next_url = self.request.build_absolute_uri()
next_url = remove_query_param(next_url, LANGUAGE_QUERY_PARAMETER)
return set_language_redirect(lang_code=lang_code, next_url=next_url)
except LookupError:
pass
share_base = kwargs.get(self.lookup_field, '')
pp = PathParser(filepath=share_base)
bucket_name, dir_base = pp.get_bucket_and_dirpath()
if not bucket_name:
return render(request, 'info.html', context={'code': 400, 'code_text': _('分享路径无效')})
# 存储桶验证和获取桶对象
h_manager = HarborManager()
try:
bucket = h_manager.get_bucket(bucket_name=bucket_name)
if not bucket:
return render(request, 'info.html', context={'code': 404, 'code_text': _('存储桶不存在')})
if dir_base:
base_obj = h_manager.get_metadata_obj(table_name=bucket.get_bucket_table_name(), path=dir_base)
if not base_obj:
return render(request, 'info.html', context={'code': 404, 'code_text': _('分享根目录不存在')})
else:
base_obj = None
# 是否有文件对象的访问权限
if not self.has_access_permission(bucket=bucket, base_dir_obj=base_obj):
return render(request, 'info.html', context={'code': 403, 'code_text': _('您没有访问权限')})
except HarborError as e:
return render(request, 'info.html', context=e.err_data_old())
# 无分享密码
if (not base_obj) or (not base_obj.has_share_password()):
return render(request, 'share.html', context={
'share_base': share_base, 'share_user': bucket.user.username, 'share_code': None})
# 验证分享密码
p = request.GET.get('p', None)
if p:
data = QueryDict(mutable=True)
data.setlist('password', [p])
form = SharePasswordForm(data) # 模拟post请求数据
if base_obj.check_share_password(p):
return render(request, 'share.html', context={
'share_base': share_base, 'share_user': bucket.user.username, 'share_code': p})
else:
form.is_valid()
form.add_error('password', error=_('分享密码有误'))
else:
form = SharePasswordForm()
content = {
'form_title': _('验证分享密码'),
'submit_text': _('确定'),
'form': form,
'share_base': share_base,
'share_user': bucket.user.username
}
return render(request, 'share_form.html', context=content)
def post(self, request, *args, **kwargs):
messages_error = []
# get advanced options
language = request.POST.get('language', translation.get_supported_language_variant(
settings.LANGUAGE_CODE))
start_row = int(request.POST.get('start_row', config.START_ROW))
header_row = int(request.POST.get('header_row', config.HEADER_ROW))
template = request.POST.get('template', config.DEFAULT_TEMPLATE)
date_format = request.POST.get('date_format', formats.get_format("SHORT_DATE_FORMAT"))
# get file and set name
excel_file = request.FILES['excel_file']
tmp_excel_name = "{}-{}-{}".format(request.user.username, time.strftime("%Y%m%d-%H%M%S"),
excel_file.name)
default_storage.save('tmp/{}'.format(tmp_excel_name), excel_file)
uploaded_wb = load_workbook(filename=excel_file)
# gets the data sheet, tries all languages
uploaded_ws = None
if _('data') in uploaded_wb.sheetnames:
uploaded_ws = uploaded_wb[_('data')]
else:
for other_lang in [lang[0] for lang in settings.LANGUAGES]:
with translation.override(other_lang):
if _('data') in uploaded_wb.sheetnames:
uploaded_ws = uploaded_wb[_('data')]
continue
if not uploaded_ws:
uploaded_ws = uploaded_wb.active
# check headers
template_obj = Template.objects.get(id=template)
mapping = getattr(template_obj, __('mapping', language))
columns_required = []
headers = [cell.value for cell in uploaded_ws[header_row]]
# checks columns from mapping exist in uploaded file
for model, fields in mapping.items():
for field_name, field_data in fields.items():
column_header = field_data['name']
if column_header not in headers:
raise Exception('Column "{}" not found, choices are: {}'
.format(column_header, ', '.join(filter(None, headers))))
if field_data['required']:
columns_required.append(column_header)
context = {}
context['columns'] = uploaded_ws[header_row]
headers = {cell.value: cell.col_idx - 1 for cell in uploaded_ws[header_row]}
uploaded_ws.delete_rows(0, amount=start_row - 1)
# map headers to columns
for model, fields in mapping.items():
for field_name, field_data in fields.items():
column_header = field_data['name']
mapping[model][field_name]['column'] = headers[column_header]
# validate rows
for row in uploaded_ws.iter_rows():
# skips empty row
if xstr(row[0].value) == '' and xstr(row[1].value) == '' and xstr(row[2].value) == '':
continue
# quick data validation
error_message = validate_data(row, mapping, start_row, date_format, language)
if error_message:
messages_error.append(error_message)
grouped_errors = {}
for row in messages_error:
row_messages = row['msgs']
for row_message in row_messages:
reference, clean_msg = row_message.split(': ', 1)
if clean_msg not in grouped_errors:
grouped_errors[clean_msg] = {}
grouped_errors[clean_msg]['reference'] = []
grouped_errors[clean_msg]['count'] = 0
grouped_errors[clean_msg]['count'] += 1
grouped_errors[clean_msg]['reference'].append(reference)
grouped_errors = sorted(grouped_errors.items(),
key=lambda k_v: k_v[1]['count'], reverse=True)
excel_data_rows = uploaded_ws
if uploaded_ws.max_row > 20:
excel_data_rows = uploaded_ws.iter_rows(min_row=1, max_row=20)
context['grouped_errors'] = grouped_errors
context['data'] = excel_data_rows
context['columns_required'] = columns_required
context['start_row'] = start_row
context['date_format'] = date_format
context['excel_file'] = tmp_excel_name
context['template'] = template
context['header_row'] = header_row
context['language'] = language
context['sheet'] = uploaded_ws.title
return render(request, self.template_name, context)
def post(self, request, *args, **kwargs):
messages_error = []
messages_info = []
imported_ids = []
counter_records_updated = 0
counter_records_created = 0
filter_type = 'iexact'
# get advanced options
language = request.POST.get('language', translation.get_supported_language_variant(
settings.LANGUAGE_CODE))
start_row = int(request.POST.get('start_row', config.START_ROW))
header_row = int(request.POST.get('header_row', config.HEADER_ROW))
template = request.POST.get('template', config.DEFAULT_TEMPLATE)
date_format = request.POST.get('date_format', formats.get_format("SHORT_DATE_FORMAT"))
# read excel file and sheet
sheet = request.POST.get('sheet', _('data'))
tmp_excel = request.POST.get('excel_file')
excel_file = default_storage.open('{}/{}'.format('tmp', tmp_excel))
uploaded_wb = load_workbook(excel_file)
uploaded_ws = uploaded_wb[sheet]
# check headers
template_obj = Template.objects.get(id=template)
headers = {cell.value: cell.col_idx - 1 for cell in uploaded_ws[header_row]}
mapping = getattr(template_obj, __('mapping', language))
# map headers to columns
for model, fields in mapping.items():
for field_name, field_data in fields.items():
column_header = field_data['name']
mapping[model][field_name]['column'] = headers[column_header]
# create log event
content = "Excel import using '{}' and template '{}'".format(tmp_excel, template)
log = Log.objects.create(module='excel import', user=request.user.username, content=content)
# import
uploaded_ws.delete_rows(0, amount=start_row - 1)
for row in uploaded_ws.iter_rows():
# skips empty row, based on first 3 values
if xstr(row[0].value) == '' and xstr(row[1].value) == '' and xstr(row[2].value) == '':
continue
# quick data validation
error_message = validate_data(row, mapping)
if error_message:
messages_error.append(error_message)
continue
# create or update contact
model = Contact
model_fields = mapping['contact']
row_dict = {}
row_dict['source_id'] = 'excel'
for field_name, field_data in model_fields.items():
value = row[field_data['column']].value
if value:
if model._meta.get_field(field_name).get_internal_type() == 'PointField':
(lat, lng) = value.split(' ')[:2]
value = Point(float(lng), float(lat))
if model._meta.get_field(field_name).get_internal_type() == 'CharField':
value = str(value)
if model._meta.get_field(field_name).get_internal_type() == 'DateField':
if not row[field_data['column']].is_date:
value = parse_date(value, date_format)
else:
value = None
# removes extra spaces if string
if isinstance(value, str):
value = xstr(value)
row_dict[field_name] = value
# there are two ways to look up a contact: name+doc and firstname+lastname+doc
if {'name', 'document'} <= set(model_fields):
contact = Contact.objects.filter(name__iexact=row_dict['name'],
document__iexact=row_dict['document)']).first()
elif {'first_name', 'last_name', 'document'} <= set(model_fields):
name = "{} {}".format(xstr(row_dict['first_name']), xstr(row_dict['last_name']))
row_dict['name'] = xstr(name)
contact = Contact.objects.filter(Q(name__iexact=row_dict['name'],
document__iexact=row_dict['document']) |
Q(first_name__iexact=row_dict['name'],
last_name__iexact=row_dict['last_name'],
document__iexact=row_dict['document'])).first()
else:
raise Exception('Mapping needs more contact data fields')
# create new organization if needed
contact_organization = Organization.objects.filter(
name__iexact=row_dict['organization']).first()
if not contact_organization and row_dict['organization']:
contact_organization = Organization.objects.filter(
varname__iexact=row_dict['organization']).first()
if not contact_organization and row_dict['organization']:
messages_info.append('Create organization: {}'.format(row_dict['organization']))
contact_organization = Organization()
contact_organization.name = row_dict['organization']
contact_organization.created_user = request.user.username
contact_organization.log = log
contact_organization.save()
# create contact if needed
if not contact:
messages_info.append('Create contact: {}'.format(contact))
contact = Contact()
if contact_organization:
contact.organization = contact_organization
counter_records_created += 1
else:
messages_info.append('Update contact: {}'.format(contact))
counter_records_updated += 1
row_dict['log'] = log
update_contact(request, contact, row_dict)
imported_ids.append(contact.id)
# create or update project contact
model = ProjectContact
model_fields = mapping['project_contact']
row_dict = {}
row_dict['source_id'] = 'excel'
for field_name, field_data in model_fields.items():
value = row[field_data['column']].value
# removes extra spaces if string
if isinstance(value, str):
value = xstr(value)
if value:
if model._meta.get_field(field_name).get_internal_type() == 'ForeignKey':
related_model = model._meta.get_field(field_name).related_model
value = try_to_find(related_model, value)
if model._meta.get_field(field_name).get_internal_type() == 'DateField':
if not row[field_data['column']].is_date:
value = parse_date(value, date_format)
else:
value = None
row_dict[field_name] = value
subproject = row_dict['subproject']
project_contact = ProjectContact.objects.filter(subproject=subproject,
contact=contact).first()
if not project_contact:
messages_info.append('Create participant: {} {}'.format(subproject, contact))
project_contact = ProjectContact()
project_contact.contact = contact
else:
messages_info.append('Update participant: {} {}'.format(subproject, contact))
row_dict['log'] = log
update_project_contact(request, project_contact, row_dict)
# gets dupes
contacts = Contact.objects.all()
if self.request.user:
contacts = contacts.for_user(self.request.user)
contacts = contacts.annotate(
name_uc=Trim(Upper(RegexpReplace(F('name'), r'\s+', ' ', 'g'))))
# manually (python) counts dupes, because count messed up the distinct() filter
names = {}
for row in contacts:
if row.name_uc not in names:
names[row.name_uc] = 0
names[row.name_uc] += 1
names_uc = list(k_v[0] for k_v in names.items() if k_v[1] > 1)
contacts_names_ids = contacts.values_list('id', flat=True).filter(name_uc__in=names_uc)
# manually (python) counts dupes, because count messed up the distinct() filter
contacts = contacts.filter(document__isnull=False).exclude(document='')
docs = {}
for row in contacts:
if row.document not in docs:
docs[row.document] = 0
docs[row.document] += 1
documents = list(k_v[0] for k_v in docs.items() if k_v[1] > 1)
contacts = Contact.objects.filter(id__in=imported_ids) \
.filter(Q(id__in=contacts_names_ids) | Q(document__in=documents)) \
.values(contact_id=F('id'),
contact_name=Coalesce('name', Value('')),
contact_sex=Coalesce('sex_id', Value('')),
contact_document=Coalesce('document', Value('')),
contact_organization=Coalesce('organization__name', Value('')),
)
context = {}
context['excel_file'] = tmp_excel
context['messages_error'] = messages_error
context['messages_info'] = messages_info
context['quantity_records_updated'] = counter_records_updated
context['quantity_records_created'] = counter_records_created
context['model'] = list(contacts)
return render(request, self.template_name, context)
