def validate(self) -> bool: valid = FlaskForm.validate(self) name = Import.get_project_by_id(self.project_id).name \ if self.project_id else '' if name != self.name.data \ and Import.get_project_by_name(self.name.data): self.name.errors.append(_('error name exists')) valid = False return valid
def import_project_update(id_: int) -> Union[str, Response]: project = Import.get_project_by_id(id_) form = ProjectForm(obj=project) form.project_id = id_ if form.validate_on_submit(): project.name = form.name.data project.description = form.description.data Import.update_project(project) flash(_('project updated'), 'info') return redirect(url_for('import_project_view', id_=project.id)) return render_template( 'display_form.html', form=form, manual_page='admin/import', title=_('import'), crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"], [_('import'), url_for('import_index')], project, _('edit')])
def get_log_for_advanced_view(entity_id: str) -> Dict[str, Any]: from openatlas.models.user import User data = Db.get_log_for_advanced_view(entity_id) return { 'creator': User.get_by_id(data['creator_id']) if data['creator_id'] else None, 'created': data['created'], 'modifier': User.get_by_id(data['modifier_id']) if data['modifier_id'] else None, 'modified': data['modified'], 'project': Import.get_project_by_id(data['project_id']) if data['project_id'] else None, 'importer': User.get_by_id(data['importer_id']) if data['importer_id'] else None, 'origin_id': data['origin_id']}
def import_index() -> str: table = Table([_('project'), _('entities'), _('description')]) for project in Import.get_all_projects(): table.rows.append( [link(project), format_number(project.count), project.description]) return render_template( 'import/index.html', table=table, title=_('import'), crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"], _('import')])
def import_project_insert() -> Union[str, Response]: form = ProjectForm() if form.validate_on_submit(): id_ = Import.insert_project(form.name.data, form.description.data) flash(_('project inserted'), 'info') return redirect(url_for('import_project_view', id_=id_)) return render_template( 'display_form.html', form=form, manual_page='admin/import', title=_('import'), crumbs=[[_('admin'), url_for('admin_index') + '#tab-data'], [_('import'), url_for('import_index')], f"+ {uc_first(_('project'))}"])
def import_project_view(id_: int) -> str: table = Table( [_('name'), _('class'), _('description'), 'origin ID', _('date')]) for entity in Entity.get_by_project_id(id_): table.rows.append([ link(entity), entity.class_.label, entity.description, entity.origin_id, format_date(entity.created) ]) project = Import.get_project_by_id(id_) return render_template( 'import/project_view.html', project=project, table=table, title=_('import'), crumbs=[[_('admin'), url_for('admin_index') + '#tab-data'], [_('import'), url_for('import_index')], project.name])
def import_project_view(id_: int) -> str: project = Import.get_project_by_id(id_) tabs = { 'info': Tab('info', content=render_template('import/project_view.html', project=project)), 'entities': Tab('entities', table=Table(['name', 'class', 'description', 'origin ID', 'date'])) } for entity in Entity.get_by_project_id(id_): tabs['entities'].table.rows.append([ link(entity), entity.class_.label, entity.description, entity.origin_id, format_date(entity.created) ]) return render_template( 'tabs.html', tabs=tabs, title=_('import'), crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"], [_('import'), url_for('import_index')], project.name])
def import_data(project_id: int, class_: str) -> str: project = Import.get_project_by_id(project_id) form = ImportForm() table = None imported = False messages: Dict[str, List[str]] = {'error': [], 'warn': []} file_data = get_backup_file_data() class_label = g.classes[class_].label if form.validate_on_submit(): file_ = request.files['file'] file_path = \ app.config['TMP_DIR'] \ / secure_filename(file_.filename) # type: ignore columns: Dict[str, List[str]] = { 'allowed': [ 'name', 'id', 'description', 'begin_from', 'begin_to', 'begin_comment', 'end_from', 'end_to', 'end_comment', 'type_ids' ], 'valid': [], 'invalid': [] } if class_ == 'place': columns['allowed'] += ['easting', 'northing'] try: file_.save(str(file_path)) data_frame = pd.read_csv(file_path, keep_default_na=False) headers = list(data_frame.columns.values) if 'name' not in headers: # pragma: no cover messages['error'].append(_('missing name column')) raise Exception() for item in headers: # pragma: no cover if item not in columns['allowed']: columns['invalid'].append(item) del data_frame[item] if columns['invalid']: # pragma: no cover messages['warn'].append( f"{_('invalid columns')}: {','.join(columns['invalid'])}") headers = list(data_frame.columns.values) # Get clean headers table_data = [] checked_data = [] origin_ids = [] names = [] missing_name_count = 0 invalid_type_ids = False invalid_geoms = False for index, row in data_frame.iterrows(): if not row['name']: # pragma: no cover missing_name_count += 1 continue table_row = [] checked_row = {} for item in headers: value = row[item] if item == 'type_ids': # pragma: no cover type_ids = [] for type_id in value.split(): if Import.check_type_id(type_id, class_): type_ids.append(type_id) else: type_ids.append( f'<span class="error">{type_id}</span>') invalid_type_ids = True value = ' '.join(type_ids) if item in ['northing', 'easting'] \ and row[item] \ and not is_float(row[item]): # pragma: no cover value = f'<span class="error">{value}</span>' invalid_geoms = True # pragma: no cover if item in [ 'begin_from', 'begin_to', 'end_from', 'end_to' ]: if not value: value = '' else: try: value = datetime64_to_timestamp( numpy.datetime64(value)) row[item] = value except ValueError: # pragma: no cover row[item] = '' if str(value) == 'NaT': value = '' else: value = \ f'<span class="error">{value}</span>' table_row.append(str(value)) checked_row[item] = row[item] if item == 'name' and form.duplicate.data: names.append(row['name'].lower()) if item == 'id' and row[item]: origin_ids.append(str(row['id'])) table_data.append(table_row) checked_data.append(checked_row) if invalid_type_ids: # pragma: no cover messages['warn'].append(_('invalid type ids')) if invalid_geoms: # pragma: no cover messages['warn'].append(_('invalid coordinates')) table = Table(headers, rows=table_data) # Checking for data inconsistency if missing_name_count: # pragma: no cover messages['warn'].append( f"{_('empty names')}: {missing_name_count}") doubles = [ item for item, count in collections.Counter(origin_ids).items() if count > 1 ] if doubles: # pragma: no cover messages['error'].append( f"{_('double IDs in import')}: {', '.join(doubles)}") existing = Import.get_origin_ids(project, origin_ids) \ if origin_ids else None if existing: messages['error'].append( f"{_('IDs already in database')}: {', '.join(existing)}") if form.duplicate.data: # Check for possible duplicates duplicates = Import.check_duplicates(class_, names) if duplicates: # pragma: no cover messages['warn'].append( f"{_('possible duplicates')}: {', '.join(duplicates)}") if messages['error']: raise Exception() except Exception: # pragma: no cover flash(_('error at import'), 'error') return render_template( 'import/import_data.html', form=form, messages=messages, file_data=file_data, title=_('import'), crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"], [_('import'), url_for('import_index')], project, class_label]) if not form.preview.data and checked_data: if not file_data['backup_too_old'] or app.config['IS_UNIT_TEST']: Transaction.begin() try: Import.import_data(project, class_, checked_data) Transaction.commit() logger.log('info', 'import', f'import: {len(checked_data)}') flash(f"{_('import of')}: {len(checked_data)}", 'info') imported = True except Exception as e: # pragma: no cover Transaction.rollback() logger.log('error', 'import', 'import failed', e) flash(_('error transaction'), 'error') return render_template( 'import/import_data.html', form=form, file_data=file_data, table=table, imported=imported, messages=messages, crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"], [_('import'), url_for('import_index')], project, class_label])
def import_project_delete(id_: int) -> Response: Import.delete_project(id_) flash(_('project deleted'), 'info') return redirect(url_for('import_index'))
def admin_index(action: Optional[str] = None, id_: Optional[int] = None) -> Union[str, Response]: if is_authorized('manager'): if id_ and action == 'delete_user': user = User.get_by_id(id_) if not user \ or user.id == current_user.id \ or (user.group == 'admin' and not is_authorized('admin')): abort(403) User.delete(id_) flash(_('user deleted'), 'info') elif action == 'remove_logo': Settings.set_logo() return redirect(f"{url_for('admin_index')}#tab-file") tables = { 'user': Table([ 'username', 'name', 'group', 'email', 'newsletter', 'created', 'last login', 'entities' ], defs=[{ 'className': 'dt-body-right', 'targets': 7 }]), 'content': Table(['name'] + list(app.config['LANGUAGES'])) } for user in User.get_all(): count = User.get_created_entities_count(user.id) email = user.email \ if is_authorized('manager') or user.settings['show_email'] else '' tables['user'].rows.append([ link(user), user.real_name, user.group, email, _('yes') if user.settings['newsletter'] else '', format_date(user.created), format_date(user.login_last_success), format_number(count) if count else '' ]) for item, languages in get_content().items(): content = [uc_first(_(item))] for language in app.config['LANGUAGES']: content.append(sanitize(languages[language], 'text')) content.append(link(_('edit'), url_for('admin_content', item=item))) tables['content'].rows.append(content) form = None if is_authorized('admin'): form = TestMailForm() if form.validate_on_submit( ) and g.settings['mail']: # pragma: no cover subject = _('Test mail from %(site_name)s', site_name=g.settings['site_name']) body = _('This test mail was sent by %(username)s', username=current_user.username) body += f" {_('at')} '{request.headers['Host']}" if send_mail(subject, body, form.receiver.data): flash( _('A test mail was sent to %(email)s.', email=form.receiver.data), 'info') else: form.receiver.data = current_user.email tabs = { 'files': Tab(_('files'), buttons=[ manual('entity/file'), button(_('edit'), url_for('admin_settings', category='files')) if is_authorized('manager') else '', button(_('list'), url_for('index', view='file')), button(_('file'), url_for('insert', class_='file')) ], content=render_template('admin/file.html', info=get_form_settings(FilesForm()), disk_space_info=get_disk_space_info())), 'user': Tab(_('user'), table=tables['user'], buttons=[ manual('admin/user'), button(_('activity'), url_for('user_activity')), button(_('newsletter'), url_for('admin_newsletter')) if is_authorized('manager') and g.settings['mail'] else '', button(_('user'), url_for('user_insert')) if is_authorized('manager') else '' ]) } if is_authorized('admin'): tabs['general'] = Tab( 'general', content=display_info(get_form_settings(GeneralForm())), buttons=[ manual('admin/general'), button(_('edit'), url_for('admin_settings', category='general')), button(_('system log'), url_for('admin_log')) ]) tabs['email'] = Tab( 'email', content=display_info(get_form_settings(MailForm())), buttons=[ manual('admin/mail'), button(_('edit'), url_for('admin_settings', category='mail')) ]) if g.settings['mail']: tabs['email'].content += display_form(form) if is_authorized('manager'): tabs['modules'] = Tab(_('modules'), content=f""" <h1>{_('Defaults for new user')}</h1> {display_info(get_form_settings(ModulesForm()))}""", buttons=[ manual('admin/modules'), button( _('edit'), url_for('admin_settings', category='modules')) ]) tabs['map'] = Tab('map', content=display_info(get_form_settings(MapForm())), buttons=[ manual('admin/map'), button(_('edit'), url_for('admin_settings', category='map')) ]) tabs['content'] = Tab('content', content=tables['content'].display(), buttons=[manual('admin/content')]) if is_authorized('contributor'): tabs['data'] = Tab('data', content=render_template( 'admin/data.html', imports=Import.get_all_projects(), info=get_form_settings(ApiForm()))) return render_template('tabs.html', tabs=tabs, title=_('admin'), crumbs=[_('admin')])
def admin_index(action: Optional[str] = None, id_: Optional[int] = None) -> Union[str, Response]: if is_authorized('manager'): if id_ and action == 'delete_user': user = User.get_by_id(id_) if not user \ or user.id == current_user.id \ or (user.group == 'admin' and not is_authorized('admin')): abort(403) # pragma: no cover User.delete(id_) flash(_('user deleted'), 'info') elif action == 'remove_logo': Settings.set_logo() return redirect(url_for('admin_index') + '#tab-file') dirs = { 'uploads': True if os.access(app.config['UPLOAD_DIR'], os.W_OK) else False, 'export/sql': True if os.access(app.config['EXPORT_DIR'] / 'sql', os.W_OK) else False, 'export/csv': True if os.access(app.config['EXPORT_DIR'] / 'csv', os.W_OK) else False } tables = { 'user': Table([ 'username', 'name', 'group', 'email', 'newsletter', 'created', 'last login', 'entities' ]), 'content': Table(['name'] + [language for language in app.config['LANGUAGES'].keys()]) } for user in User.get_all(): count = User.get_created_entities_count(user.id) email = user.email if is_authorized( 'manager') or user.settings['show_email'] else '' tables['user'].rows.append([ link(user), user.real_name, user.group, email, _('yes') if user.settings['newsletter'] else '', format_date(user.created), format_date(user.login_last_success), format_number(count) if count else '' ]) for item, languages in Content.get_content().items(): content = [uc_first(_(item))] for language in app.config['LANGUAGES'].keys(): content.append(sanitize(languages[language], 'text')) content.append(link(_('edit'), url_for('admin_content', item=item))) tables['content'].rows.append(content) form = None if is_authorized('admin'): form = TestMailForm() if form.validate_on_submit( ) and session['settings']['mail']: # pragma: no cover subject = _('Test mail from %(site_name)s', site_name=session['settings']['site_name']) body = _('This test mail was sent by %(username)s', username=current_user.username) body += ' ' + _('at') + ' ' + request.headers['Host'] if send_mail(subject, body, form.receiver.data): flash( _('A test mail was sent to %(email)s.', email=form.receiver.data), 'info') else: form.receiver.data = current_user.email return render_template('admin/index.html', form=form, tables=tables, settings=session['settings'], writeable_dirs=dirs, disk_space_info=get_disk_space_info(), imports=Import.get_all_projects(), title=_('admin'), crumbs=[_('admin')], info={ 'file': get_form_settings(FilesForm()), 'general': get_form_settings(GeneralForm()), 'mail': get_form_settings(MailForm()), 'map': get_form_settings(MapForm()), 'api': get_form_settings(ApiForm()), 'modules': get_form_settings(ModulesForm()) })