def validate(self) -> bool:
valid = FlaskForm.validate(self)
name = Import.get_project_by_id(self.project_id).name \
if self.project_id else ''
if name != self.name.data \
and Import.get_project_by_name(self.name.data):
self.name.errors.append(_('error name exists'))
valid = False
return valid
def import_project_update(id_: int) -> Union[str, Response]:
project = Import.get_project_by_id(id_)
form = ProjectForm(obj=project)
form.project_id = id_
if form.validate_on_submit():
project.name = form.name.data
project.description = form.description.data
Import.update_project(project)
flash(_('project updated'), 'info')
return redirect(url_for('import_project_view', id_=project.id))
return render_template(
'display_form.html',
form=form,
manual_page='admin/import',
title=_('import'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('import'), url_for('import_index')], project,
_('edit')])
def get_log_for_advanced_view(entity_id: str) -> Dict[str, Any]:
from openatlas.models.user import User
data = Db.get_log_for_advanced_view(entity_id)
return {
'creator': User.get_by_id(data['creator_id']) if data['creator_id'] else None,
'created': data['created'],
'modifier': User.get_by_id(data['modifier_id']) if data['modifier_id'] else None,
'modified': data['modified'],
'project': Import.get_project_by_id(data['project_id']) if data['project_id'] else None,
'importer': User.get_by_id(data['importer_id']) if data['importer_id'] else None,
'origin_id': data['origin_id']}
def import_index() -> str:
table = Table([_('project'), _('entities'), _('description')])
for project in Import.get_all_projects():
table.rows.append(
[link(project),
format_number(project.count), project.description])
return render_template(
'import/index.html',
table=table,
title=_('import'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
_('import')])
def import_project_insert() -> Union[str, Response]:
form = ProjectForm()
if form.validate_on_submit():
id_ = Import.insert_project(form.name.data, form.description.data)
flash(_('project inserted'), 'info')
return redirect(url_for('import_project_view', id_=id_))
return render_template(
'display_form.html',
form=form,
manual_page='admin/import',
title=_('import'),
crumbs=[[_('admin'), url_for('admin_index') + '#tab-data'],
[_('import'), url_for('import_index')],
f"+ {uc_first(_('project'))}"])
def import_project_view(id_: int) -> str:
table = Table(
[_('name'),
_('class'),
_('description'), 'origin ID',
_('date')])
for entity in Entity.get_by_project_id(id_):
table.rows.append([
link(entity), entity.class_.label, entity.description,
entity.origin_id,
format_date(entity.created)
])
project = Import.get_project_by_id(id_)
return render_template(
'import/project_view.html',
project=project,
table=table,
title=_('import'),
crumbs=[[_('admin'), url_for('admin_index') + '#tab-data'],
[_('import'), url_for('import_index')], project.name])
def import_project_view(id_: int) -> str:
project = Import.get_project_by_id(id_)
tabs = {
'info':
Tab('info',
content=render_template('import/project_view.html',
project=project)),
'entities':
Tab('entities',
table=Table(['name', 'class', 'description', 'origin ID', 'date']))
}
for entity in Entity.get_by_project_id(id_):
tabs['entities'].table.rows.append([
link(entity), entity.class_.label, entity.description,
entity.origin_id,
format_date(entity.created)
])
return render_template(
'tabs.html',
tabs=tabs,
title=_('import'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('import'), url_for('import_index')], project.name])
def import_data(project_id: int, class_: str) -> str:
project = Import.get_project_by_id(project_id)
form = ImportForm()
table = None
imported = False
messages: Dict[str, List[str]] = {'error': [], 'warn': []}
file_data = get_backup_file_data()
class_label = g.classes[class_].label
if form.validate_on_submit():
file_ = request.files['file']
file_path = \
app.config['TMP_DIR'] \
/ secure_filename(file_.filename) # type: ignore
columns: Dict[str, List[str]] = {
'allowed': [
'name', 'id', 'description', 'begin_from', 'begin_to',
'begin_comment', 'end_from', 'end_to', 'end_comment',
'type_ids'
],
'valid': [],
'invalid': []
}
if class_ == 'place':
columns['allowed'] += ['easting', 'northing']
try:
file_.save(str(file_path))
data_frame = pd.read_csv(file_path, keep_default_na=False)
headers = list(data_frame.columns.values)
if 'name' not in headers: # pragma: no cover
messages['error'].append(_('missing name column'))
raise Exception()
for item in headers: # pragma: no cover
if item not in columns['allowed']:
columns['invalid'].append(item)
del data_frame[item]
if columns['invalid']: # pragma: no cover
messages['warn'].append(
f"{_('invalid columns')}: {','.join(columns['invalid'])}")
headers = list(data_frame.columns.values) # Get clean headers
table_data = []
checked_data = []
origin_ids = []
names = []
missing_name_count = 0
invalid_type_ids = False
invalid_geoms = False
for index, row in data_frame.iterrows():
if not row['name']: # pragma: no cover
missing_name_count += 1
continue
table_row = []
checked_row = {}
for item in headers:
value = row[item]
if item == 'type_ids': # pragma: no cover
type_ids = []
for type_id in value.split():
if Import.check_type_id(type_id, class_):
type_ids.append(type_id)
else:
type_ids.append(
f'<span class="error">{type_id}</span>')
invalid_type_ids = True
value = ' '.join(type_ids)
if item in ['northing', 'easting'] \
and row[item] \
and not is_float(row[item]): # pragma: no cover
value = f'<span class="error">{value}</span>'
invalid_geoms = True # pragma: no cover
if item in [
'begin_from', 'begin_to', 'end_from', 'end_to'
]:
if not value:
value = ''
else:
try:
value = datetime64_to_timestamp(
numpy.datetime64(value))
row[item] = value
except ValueError: # pragma: no cover
row[item] = ''
if str(value) == 'NaT':
value = ''
else:
value = \
f'<span class="error">{value}</span>'
table_row.append(str(value))
checked_row[item] = row[item]
if item == 'name' and form.duplicate.data:
names.append(row['name'].lower())
if item == 'id' and row[item]:
origin_ids.append(str(row['id']))
table_data.append(table_row)
checked_data.append(checked_row)
if invalid_type_ids: # pragma: no cover
messages['warn'].append(_('invalid type ids'))
if invalid_geoms: # pragma: no cover
messages['warn'].append(_('invalid coordinates'))
table = Table(headers, rows=table_data)
# Checking for data inconsistency
if missing_name_count: # pragma: no cover
messages['warn'].append(
f"{_('empty names')}: {missing_name_count}")
doubles = [
item
for item, count in collections.Counter(origin_ids).items()
if count > 1
]
if doubles: # pragma: no cover
messages['error'].append(
f"{_('double IDs in import')}: {', '.join(doubles)}")
existing = Import.get_origin_ids(project, origin_ids) \
if origin_ids else None
if existing:
messages['error'].append(
f"{_('IDs already in database')}: {', '.join(existing)}")
if form.duplicate.data: # Check for possible duplicates
duplicates = Import.check_duplicates(class_, names)
if duplicates: # pragma: no cover
messages['warn'].append(
f"{_('possible duplicates')}: {', '.join(duplicates)}")
if messages['error']:
raise Exception()
except Exception: # pragma: no cover
flash(_('error at import'), 'error')
return render_template(
'import/import_data.html',
form=form,
messages=messages,
file_data=file_data,
title=_('import'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('import'), url_for('import_index')], project,
class_label])
if not form.preview.data and checked_data:
if not file_data['backup_too_old'] or app.config['IS_UNIT_TEST']:
Transaction.begin()
try:
Import.import_data(project, class_, checked_data)
Transaction.commit()
logger.log('info', 'import',
f'import: {len(checked_data)}')
flash(f"{_('import of')}: {len(checked_data)}", 'info')
imported = True
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'import', 'import failed', e)
flash(_('error transaction'), 'error')
return render_template(
'import/import_data.html',
form=form,
file_data=file_data,
table=table,
imported=imported,
messages=messages,
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('import'), url_for('import_index')], project, class_label])
def import_project_delete(id_: int) -> Response:
Import.delete_project(id_)
flash(_('project deleted'), 'info')
return redirect(url_for('import_index'))
def admin_index(action: Optional[str] = None,
id_: Optional[int] = None) -> Union[str, Response]:
if is_authorized('manager'):
if id_ and action == 'delete_user':
user = User.get_by_id(id_)
if not user \
or user.id == current_user.id \
or (user.group == 'admin' and not is_authorized('admin')):
abort(403)
User.delete(id_)
flash(_('user deleted'), 'info')
elif action == 'remove_logo':
Settings.set_logo()
return redirect(f"{url_for('admin_index')}#tab-file")
tables = {
'user':
Table([
'username', 'name', 'group', 'email', 'newsletter', 'created',
'last login', 'entities'
],
defs=[{
'className': 'dt-body-right',
'targets': 7
}]),
'content':
Table(['name'] + list(app.config['LANGUAGES']))
}
for user in User.get_all():
count = User.get_created_entities_count(user.id)
email = user.email \
if is_authorized('manager') or user.settings['show_email'] else ''
tables['user'].rows.append([
link(user), user.real_name, user.group, email,
_('yes') if user.settings['newsletter'] else '',
format_date(user.created),
format_date(user.login_last_success),
format_number(count) if count else ''
])
for item, languages in get_content().items():
content = [uc_first(_(item))]
for language in app.config['LANGUAGES']:
content.append(sanitize(languages[language], 'text'))
content.append(link(_('edit'), url_for('admin_content', item=item)))
tables['content'].rows.append(content)
form = None
if is_authorized('admin'):
form = TestMailForm()
if form.validate_on_submit(
) and g.settings['mail']: # pragma: no cover
subject = _('Test mail from %(site_name)s',
site_name=g.settings['site_name'])
body = _('This test mail was sent by %(username)s',
username=current_user.username)
body += f" {_('at')} '{request.headers['Host']}"
if send_mail(subject, body, form.receiver.data):
flash(
_('A test mail was sent to %(email)s.',
email=form.receiver.data), 'info')
else:
form.receiver.data = current_user.email
tabs = {
'files':
Tab(_('files'),
buttons=[
manual('entity/file'),
button(_('edit'), url_for('admin_settings', category='files'))
if is_authorized('manager') else '',
button(_('list'), url_for('index', view='file')),
button(_('file'), url_for('insert', class_='file'))
],
content=render_template('admin/file.html',
info=get_form_settings(FilesForm()),
disk_space_info=get_disk_space_info())),
'user':
Tab(_('user'),
table=tables['user'],
buttons=[
manual('admin/user'),
button(_('activity'), url_for('user_activity')),
button(_('newsletter'), url_for('admin_newsletter'))
if is_authorized('manager') and g.settings['mail'] else '',
button(_('user'), url_for('user_insert'))
if is_authorized('manager') else ''
])
}
if is_authorized('admin'):
tabs['general'] = Tab(
'general',
content=display_info(get_form_settings(GeneralForm())),
buttons=[
manual('admin/general'),
button(_('edit'), url_for('admin_settings',
category='general')),
button(_('system log'), url_for('admin_log'))
])
tabs['email'] = Tab(
'email',
content=display_info(get_form_settings(MailForm())),
buttons=[
manual('admin/mail'),
button(_('edit'), url_for('admin_settings', category='mail'))
])
if g.settings['mail']:
tabs['email'].content += display_form(form)
if is_authorized('manager'):
tabs['modules'] = Tab(_('modules'),
content=f"""
<h1>{_('Defaults for new user')}</h1>
{display_info(get_form_settings(ModulesForm()))}""",
buttons=[
manual('admin/modules'),
button(
_('edit'),
url_for('admin_settings',
category='modules'))
])
tabs['map'] = Tab('map',
content=display_info(get_form_settings(MapForm())),
buttons=[
manual('admin/map'),
button(_('edit'),
url_for('admin_settings', category='map'))
])
tabs['content'] = Tab('content',
content=tables['content'].display(),
buttons=[manual('admin/content')])
if is_authorized('contributor'):
tabs['data'] = Tab('data',
content=render_template(
'admin/data.html',
imports=Import.get_all_projects(),
info=get_form_settings(ApiForm())))
return render_template('tabs.html',
tabs=tabs,
title=_('admin'),
crumbs=[_('admin')])
def admin_index(action: Optional[str] = None,
id_: Optional[int] = None) -> Union[str, Response]:
if is_authorized('manager'):
if id_ and action == 'delete_user':
user = User.get_by_id(id_)
if not user \
or user.id == current_user.id \
or (user.group == 'admin' and not is_authorized('admin')):
abort(403) # pragma: no cover
User.delete(id_)
flash(_('user deleted'), 'info')
elif action == 'remove_logo':
Settings.set_logo()
return redirect(url_for('admin_index') + '#tab-file')
dirs = {
'uploads':
True if os.access(app.config['UPLOAD_DIR'], os.W_OK) else False,
'export/sql':
True if os.access(app.config['EXPORT_DIR'] /
'sql', os.W_OK) else False,
'export/csv':
True if os.access(app.config['EXPORT_DIR'] / 'csv', os.W_OK) else False
}
tables = {
'user':
Table([
'username', 'name', 'group', 'email', 'newsletter', 'created',
'last login', 'entities'
]),
'content':
Table(['name'] +
[language for language in app.config['LANGUAGES'].keys()])
}
for user in User.get_all():
count = User.get_created_entities_count(user.id)
email = user.email if is_authorized(
'manager') or user.settings['show_email'] else ''
tables['user'].rows.append([
link(user), user.real_name, user.group, email,
_('yes') if user.settings['newsletter'] else '',
format_date(user.created),
format_date(user.login_last_success),
format_number(count) if count else ''
])
for item, languages in Content.get_content().items():
content = [uc_first(_(item))]
for language in app.config['LANGUAGES'].keys():
content.append(sanitize(languages[language], 'text'))
content.append(link(_('edit'), url_for('admin_content', item=item)))
tables['content'].rows.append(content)
form = None
if is_authorized('admin'):
form = TestMailForm()
if form.validate_on_submit(
) and session['settings']['mail']: # pragma: no cover
subject = _('Test mail from %(site_name)s',
site_name=session['settings']['site_name'])
body = _('This test mail was sent by %(username)s',
username=current_user.username)
body += ' ' + _('at') + ' ' + request.headers['Host']
if send_mail(subject, body, form.receiver.data):
flash(
_('A test mail was sent to %(email)s.',
email=form.receiver.data), 'info')
else:
form.receiver.data = current_user.email
return render_template('admin/index.html',
form=form,
tables=tables,
settings=session['settings'],
writeable_dirs=dirs,
disk_space_info=get_disk_space_info(),
imports=Import.get_all_projects(),
title=_('admin'),
crumbs=[_('admin')],
info={
'file': get_form_settings(FilesForm()),
'general': get_form_settings(GeneralForm()),
'mail': get_form_settings(MailForm()),
'map': get_form_settings(MapForm()),
'api': get_form_settings(ApiForm()),
'modules': get_form_settings(ModulesForm())
})