def insert_files(form: FlaskForm,
origin: Optional[Entity] = None) -> Union[str, Response]:
filenames = []
url = url_for('index', view=g.classes['file'].view)
try:
Transaction.begin()
entity_name = form.name.data.strip()
for count, file in enumerate(form.file.data):
entity = Entity.insert('file', file.filename)
url = get_redirect_url(form, entity, origin)
# Add 'a' to prevent emtpy temporary filename, has no side effects
filename = secure_filename(f'a{file.filename}')
new_name = f"{entity.id}.{filename.rsplit('.', 1)[1].lower()}"
file.save(str(app.config['UPLOAD_DIR'] / new_name))
filenames.append(new_name)
if g.settings['image_processing']:
resize_image(new_name)
if len(form.file.data) > 1:
form.name.data = f'{entity_name}_{str(count + 1).zfill(2)}'
if origin:
url = f"{url_for('view', id_=origin.id)}#tab-file"
entity.update(process_form_data(form, entity, origin))
logger.log_user(entity.id, 'insert')
Transaction.commit()
flash(_('entity created'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
for filename in filenames:
(app.config['UPLOAD_DIR'] / filename).unlink()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
url = url_for('index', view=g.classes['file'].view)
return url
def admin_settings(category: str) -> Union[str, Response]:
if category in ['general', 'mail'] and not is_authorized('admin'):
abort(403) # pragma: no cover
form = getattr(importlib.import_module('openatlas.forms.setting'),
uc_first(category) + 'Form')() # Get forms dynamically
if form.validate_on_submit():
Transaction.begin()
try:
Settings.update(form)
logger.log('info', 'settings', 'Settings updated')
Transaction.commit()
flash(_('info update'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
tab = 'data' if category == 'api' else category
tab = 'email' if category == 'mail' else tab
return redirect(url_for('admin_index') + '#tab-' + tab)
set_form_settings(form)
return render_template('display_form.html',
form=form,
manual_page='admin/' + category,
title=_('admin'),
crumbs=[[
_('admin'),
url_for('admin_index') + '#tab-' +
('data' if category == 'api' else category)
],
_(category)])
def sql_execute() -> str:
file_data = get_backup_file_data()
response = ''
form = SqlForm()
if form.validate_on_submit() and not file_data['backup_too_old']:
Transaction.begin()
try:
g.cursor.execute(form.statement.data)
response = f'<p>Rows affected: {g.cursor.rowcount}</p>'
try:
response += f'<p>{g.cursor.fetchall()}</p>'
except Exception: # pragma: no cover
pass # Assuming no SELECT statement so returning rowcount
Transaction.commit()
flash(_('SQL executed'), 'info')
logger.log('info', 'database', 'SQL executed', form.statement.data)
except Exception as e:
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
response = str(e)
flash(_('error transaction'), 'error')
return render_template(
'sql/execute.html',
form=form,
response=response,
file_data=file_data,
title=_('SQL'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('SQL'), url_for('sql_index')],
_('execute')])
def save(form: FlaskForm,
entity: Optional[Entity] = None,
class_: Optional[str] = None,
origin: Optional[Entity] = None) -> Union[str, Response]:
if class_ == 'file' and not entity:
return insert_file(form, origin)
Transaction.begin()
action = 'update'
try:
if not entity:
action = 'insert'
entity = insert_entity(form, class_, origin)
if isinstance(entity, ReferenceSystem):
entity.name = entity.name \
if hasattr(entity, 'system') and entity.system \
else form.name.data
entity.description = form.description.data
entity.website_url = form.website_url.data \
if form.website_url.data else None
entity.resolver_url = form.resolver_url.data \
if form.resolver_url.data else None
entity.placeholder = form.placeholder.data \
if form.placeholder.data else None
entity.update_system(form)
if hasattr(form, 'forms'):
entity.add_forms(form)
else:
entity.update(form)
class_ = entity.class_.name
update_links(entity, form, action, origin)
url = link_and_get_redirect_url(form, entity, class_, origin)
logger.log_user(entity.id, action)
Transaction.commit()
flash(
_('entity created') if action == 'insert' else _('info update'),
'info')
except InvalidGeomException as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database',
'transaction failed because of invalid geom', e)
flash(_('Invalid geom entered'), 'error')
if action == 'update' and entity:
url = url_for('update',
id_=entity.id,
origin_id=origin.id if origin else None)
else:
url = url_for('index', view=g.classes[class_].view)
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
if action == 'update' and entity:
url = url_for('update',
id_=entity.id,
origin_id=origin.id if origin else None)
else:
url = url_for('index', view=g.classes[class_].view)
if class_ in ['administrative_unit', 'type']:
url = url_for('node_index')
return url
def profile_settings(category: str) -> Union[str, Response]:
if category not in ['profile', 'display'] and not is_authorized('contributor'):
abort(403) # pragma: no cover
form = getattr(
importlib.import_module('openatlas.forms.setting'),
uc_first(category) + 'Form')()
if form.validate_on_submit():
for field in form:
if field.type in ['CSRFTokenField', 'HiddenField', 'SubmitField']:
continue
if field.name == 'name':
current_user.real_name = field.data
elif field.name == 'email':
current_user.email = field.data
else:
current_user.settings[field.name] = field.data
Transaction.begin()
try:
current_user.update()
current_user.update_settings(form)
Transaction.commit()
session['language'] = current_user.settings['language']
flash(_('info update'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return redirect(url_for('profile_index') + '#tab-' + category)
set_form_settings(form, True)
return render_template(
'display_form.html',
form=form,
manual_page='profile',
title=_('profile'),
crumbs=[[_('profile'), url_for('profile_index') + '#tab-' + category], _(category)])
def hierarchy_insert(category: str) -> Union[str, Response]:
form = build_form('hierarchy', code=category)
form.classes.choices = Type.get_class_choices()
if form.validate_on_submit():
if Type.check_hierarchy_exists(form.name.data):
flash(_('error name exists'), 'error')
return render_template('display_form.html', form=form)
try:
Transaction.begin()
type_ = Entity.insert('type', sanitize(form.name.data))
Type.insert_hierarchy(
type_, # type: ignore
category,
form.classes.data,
is_multiple(form, category))
type_.update(process_form_data(form, type_))
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
abort(418)
flash(_('entity created'), 'info')
return redirect(f"{url_for('type_index')}#menu-tab-{category}")
return render_template(
'display_form.html',
form=form,
manual_page='entity/type',
title=_('types'),
crumbs=[
[_('types'), url_for('type_index')],
f'+ {uc_first(_(category))}'])
def member_update(id_: int, origin_id: int) -> Union[str, Response]:
link_ = Link.get_by_id(id_)
domain = Entity.get_by_id(link_.domain.id)
range_ = Entity.get_by_id(link_.range.id)
origin = range_ if origin_id == range_.id else domain
form = build_form('actor_function', link_)
if form.validate_on_submit():
Transaction.begin()
try:
link_.delete()
link_ = Link.get_by_id(
domain.link('P107', range_, form.description.data)[0])
link_.set_dates(process_form_dates(form))
link_.type = get_link_type(form)
link_.update()
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return redirect(
f"{url_for('view', id_=origin.id)}"
f"#tab-member{'-of' if origin.id == range_.id else ''}")
form.save.label.text = _('save')
related = range_ if origin_id == domain.id else domain
return render_template(
'display_form.html',
form=form,
crumbs=[[_('actor'), url_for('index', view='actor')], origin, related,
_('edit')])
def relation_insert(origin_id: int) -> Union[str, Response]:
origin = Entity.get_by_id(origin_id)
form = build_form('actor_actor_relation')
form.relation_origin_id.data = origin.id
if form.validate_on_submit():
Transaction.begin()
try:
for actor in Entity.get_by_ids(ast.literal_eval(form.actor.data)):
if form.inverse.data:
link_ = Link.get_by_id(
actor.link('OA7', origin, form.description.data)[0])
else:
link_ = Link.get_by_id(
origin.link('OA7', actor, form.description.data)[0])
link_.set_dates(form)
link_.type = get_link_type(form)
link_.update()
Transaction.commit()
flash(_('entity created'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
if hasattr(form, 'continue_') and form.continue_.data == 'yes':
return redirect(url_for('relation_insert', origin_id=origin_id))
return redirect(
url_for('entity_view', id_=origin.id) + '#tab-relation')
return render_template(
'display_form.html',
form=form,
title=_('relation'),
crumbs=[[_('actor'), url_for('index', view='actor')], origin,
'+ ' + uc_first(_('relation'))])
def relation_update(link_: Link, domain: Entity, range_: Entity,
origin: Entity) -> Union[str, Response]:
origin = range_ if origin.id == range_.id else domain
related = range_ if origin.id == domain.id else domain
form = build_form('actor_actor_relation', link_)
if form.validate_on_submit():
Transaction.begin()
try:
link_.delete()
if form.inverse.data:
link_ = Link.get_by_id(
related.link('OA7', origin, form.description.data)[0])
else:
link_ = Link.get_by_id(
origin.link('OA7', related, form.description.data)[0])
link_.set_dates(process_form_dates(form))
link_.type = get_link_type(form)
link_.update()
Transaction.commit()
flash(_('info update'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return redirect(f"{url_for('view', id_=origin.id)}#tab-relation")
if origin.id == range_.id:
form.inverse.data = True
return render_template(
'display_form.html',
form=form,
title=_('relation'),
crumbs=[[_('actor'), url_for('index', view='actor')], origin, related,
_('edit')])
def hierarchy_update(id_: int) -> Union[str, Response]:
hierarchy = g.types[id_]
if hierarchy.category in ('standard', 'system'):
abort(403)
form = build_form('hierarchy', hierarchy)
form.classes.choices = Type.get_class_choices(hierarchy)
linked_entities = set()
has_multiple_links = False
for entity in get_entities_linked_to_type_recursive(id_, []):
if entity.id in linked_entities:
has_multiple_links = True
break
linked_entities.add(entity.id)
if hasattr(form, 'multiple') and has_multiple_links:
form.multiple.render_kw = {'disabled': 'disabled'}
if form.validate_on_submit():
if form.name.data != hierarchy.name and Type.get_types(form.name.data):
flash(_('error name exists'), 'error')
else:
Transaction.begin()
try:
Type.update_hierarchy(hierarchy,
sanitize(form.name.data),
form.classes.data,
multiple=(hierarchy.category == 'value'
or (hasattr(form, 'multiple')
and form.multiple.data)
or has_multiple_links))
hierarchy.update(process_form_data(form, hierarchy))
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
abort(418)
flash(_('info update'), 'info')
tab = 'value' if g.types[id_].category == 'value' else 'custom'
return redirect(
f"{url_for('type_index')}#menu-tab-{tab}_collapse-{hierarchy.id}")
form.multiple = hierarchy.multiple
table = Table(paging=False)
for class_name in hierarchy.classes:
count = Type.get_form_count(hierarchy, class_name)
table.rows.append([
g.classes[class_name].label,
format_number(count) if count else link(
_('remove'),
url_for(
'remove_class', id_=hierarchy.id, class_name=class_name))
])
return render_template('display_form.html',
form=form,
table=table,
manual_page='entity/type',
title=_('types'),
crumbs=[[_('types'),
url_for('type_index')], hierarchy,
_('edit')])
def sex_update(id_: int) -> Union[str, Response]:
class Form(FlaskForm):
pass
entity = Entity.get_by_id(id_, types=True)
choices = [(option, option) for option in SexEstimation.options]
for feature, values in SexEstimation.features.items():
description = ''
if values['female'] or values['male']:
description = f"Female: {values['female']}, male: {values['male']}"
setattr(
Form, feature,
SelectField(
f"{uc_first(feature.replace('_', ' '))} ({values['category']})",
choices=choices,
default='Not preserved',
description=description))
setattr(Form, 'save', SubmitField(_('save')))
form = Form()
types = get_types(entity.id)
if form.validate_on_submit():
data = form.data
data.pop('save', None)
data.pop('csrf_token', None)
try:
Transaction.begin()
SexEstimation.save(entity, data, types)
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return redirect(url_for('sex', id_=entity.id))
# Fill in data
for dict_ in types:
getattr(form, g.types[dict_['id']].name).data = dict_['description']
return render_template(
'display_form.html',
entity=entity,
manual_page='tools/anthropological_analyses',
form=form,
crumbs=[
entity,
[
_('anthropological analyses'),
url_for('anthropology_index', id_=entity.id)
], [_('sex estimation'),
url_for('sex', id_=entity.id)],
_('edit')
])
def involvement_insert(origin_id: int) -> Union[str, Response]:
origin = Entity.get_by_id(origin_id)
form = build_form('involvement', origin=origin)
form.activity.choices = [('P11', g.properties['P11'].name_inverse)]
if origin.class_.name in ['acquisition', 'activity', 'production']:
form.activity.choices.append(('P14', g.properties['P14'].name_inverse))
if origin.class_.name == 'acquisition':
form.activity.choices.append(
('P22', g.properties['P22'].name_inverse))
form.activity.choices.append(
('P23', g.properties['P23'].name_inverse))
if form.validate_on_submit():
Transaction.begin()
try:
if origin.class_.view == 'event':
for actor in Entity.get_by_ids(
ast.literal_eval(form.actor.data)):
link_ = Link.get_by_id(
origin.link(form.activity.data, actor,
form.description.data)[0])
link_.set_dates(process_form_dates(form))
link_.type = get_link_type(form)
link_.update()
else:
for event in Entity.get_by_ids(
ast.literal_eval(form.event.data)):
link_ = Link.get_by_id(
event.link(form.activity.data, origin,
form.description.data)[0])
link_.set_dates(process_form_dates(form))
link_.type = get_link_type(form)
link_.update()
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
if hasattr(form, 'continue_') and form.continue_.data == 'yes':
return redirect(url_for('involvement_insert', origin_id=origin_id))
return redirect(
f"{url_for('view', id_=origin.id)}"
f"#tab-{'actor' if origin.class_.view == 'event' else 'event'}")
return render_template('display_form.html',
form=form,
crumbs=[[
_(origin.class_.view),
url_for('index', view=origin.class_.view)
], origin,
_('involvement')])
def save(
form: FlaskForm,
entity: Optional[Entity] = None,
class_: Optional[str] = None,
origin: Optional[Entity] = None) -> Union[str, Response]:
Transaction.begin()
action = 'update' if entity else 'insert'
try:
if not entity:
if not class_:
abort(404) # pragma: no cover, entity or class needed
entity = insert_entity(form, class_)
if class_ == 'source_translation' and origin:
origin.link('P73', entity)
redirect_link_id = entity.update(
data=process_form_data(form, entity, origin),
new=(action == 'insert'))
logger.log_user(entity.id, action)
Transaction.commit()
url = get_redirect_url(form, entity, origin, redirect_link_id)
flash(
_('entity created') if action == 'insert' else _('info update'),
'info')
except InvalidGeomException as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'invalid geom', e)
flash(_('Invalid geom entered'), 'error')
url = url_for('index', view=g.classes[class_].view)
if action == 'update' and entity:
url = url_for(
'update',
id_=entity.id,
origin_id=origin.id if origin else None)
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
if action == 'update' and entity:
url = url_for(
'update',
id_=entity.id,
origin_id=origin.id if origin else None)
else:
url = url_for('index', view=g.classes[class_].view)
if class_ in ['administrative_unit', 'type']:
url = url_for('type_index')
return url
def save(form: FlaskForm,
node: Optional[Node] = None,
param: Optional[str] = None) -> Optional[Node]:
Transaction.begin()
try:
if node:
Node.update_hierarchy(node, form)
else:
node = Entity.insert('type', sanitize(form.name.data))
Node.insert_hierarchy(node, form, value_type=(param == 'value'))
node.update(form)
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
abort(418)
return node
def involvement_update(link_: Link, origin: Entity) -> Union[str, Response]:
form = build_form('involvement', link_)
form.activity.choices = [('P11', g.properties['P11'].name)]
event = Entity.get_by_id(link_.domain.id)
actor = Entity.get_by_id(link_.range.id)
origin = event if origin.id == event.id else actor
if event.class_.name in ['acquisition', 'activity']:
form.activity.choices.append(('P14', g.properties['P14'].name))
if event.class_.name == 'acquisition':
form.activity.choices.append(('P22', g.properties['P22'].name))
form.activity.choices.append(('P23', g.properties['P23'].name))
if form.validate_on_submit():
Transaction.begin()
try:
link_.delete()
link_ = Link.get_by_id(
event.link(
form.activity.data,
actor,
form.description.data)[0])
link_.set_dates(process_form_dates(form))
link_.type = get_link_type(form)
link_.update()
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return redirect(
f"{url_for('view', id_=origin.id)}"
f"#tab-{'actor' if origin.class_.view == 'event' else 'event'}")
form.save.label.text = _('save')
form.activity.data = link_.property.code
form.description.data = link_.description
return render_template(
'display_form.html',
origin=origin,
form=form,
crumbs=[
[_(origin.class_.view), url_for('index', view=origin.class_.view)],
origin,
event if origin.id != event.id else actor,
_('edit')])
def save(form: FlaskForm,
entity: Optional[Entity] = None,
source: Optional[Entity] = None) -> Entity:
Transaction.begin()
try:
if entity:
logger.log_user(entity.id, 'update')
elif source:
entity = Entity.insert('source_translation', form.name.data)
source.link('P73', entity)
logger.log_user(entity.id, 'insert')
else:
abort(400) # pragma: no cover, entity or source needed
entity.update(form)
Transaction.commit()
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return entity # type: ignore
def admin_settings(category: str) -> Union[str, Response]:
if category in ['general', 'mail'] and not is_authorized('admin'):
abort(403) # pragma: no cover
form_name = f"{uc_first(category)}Form"
form = getattr(
importlib.import_module('openatlas.forms.setting'),
form_name)()
if form.validate_on_submit():
data = {}
for field in form:
if field.type in ['CSRFTokenField', 'HiddenField', 'SubmitField']:
continue
value = field.data
if field.type == 'BooleanField':
value = 'True' if field.data else ''
data[field.name] = value
Transaction.begin()
try:
Settings.update(data)
logger.log('info', 'settings', 'Settings updated')
Transaction.commit()
flash(_('info update'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
return redirect(
f"{url_for('admin_index')}"
f"#tab-{category.replace('api', 'data').replace('mail', 'email')}")
set_form_settings(form)
return render_template(
'display_form.html',
form=form,
manual_page=f"admin/{category}",
title=_('admin'),
crumbs=[
[
_('admin'),
f"{url_for('admin_index')}"
f"#tab-{'data' if category == 'api' else category}"],
_(category)])
def member_insert(origin_id: int,
code: str = 'member') -> Union[str, Response]:
origin = Entity.get_by_id(origin_id)
form = build_form('actor_function', code=code)
form.member_origin_id.data = origin.id
if form.validate_on_submit():
Transaction.begin()
try:
member_field = getattr(form, 'actor') \
if code == 'member' else getattr(form, 'group')
for actor in Entity.get_by_ids(ast.literal_eval(
member_field.data)):
if code == 'membership':
link_ = Link.get_by_id(
actor.link('P107', origin, form.description.data)[0])
else:
link_ = Link.get_by_id(
origin.link('P107', actor, form.description.data)[0])
link_.set_dates(process_form_dates(form))
link_.type = get_link_type(form)
link_.update()
Transaction.commit()
flash(_('entity created'), 'info')
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'database', 'transaction failed', e)
flash(_('error transaction'), 'error')
if hasattr(form, 'continue_') and form.continue_.data == 'yes':
return redirect(
url_for('member_insert', origin_id=origin_id, code=code))
return redirect(f"{url_for('view', id_=origin.id)}"
f"#tab-member{'' if code == 'member' else '-of'}")
return render_template(
'display_form.html',
form=form,
crumbs=[[_('actor'), url_for('index', view='actor')], origin,
_('member')])
def import_data(project_id: int, class_: str) -> str:
project = Import.get_project_by_id(project_id)
form = ImportForm()
table = None
imported = False
messages: Dict[str, List[str]] = {'error': [], 'warn': []}
file_data = get_backup_file_data()
class_label = g.classes[class_].label
if form.validate_on_submit():
file_ = request.files['file']
file_path = \
app.config['TMP_DIR'] \
/ secure_filename(file_.filename) # type: ignore
columns: Dict[str, List[str]] = {
'allowed': [
'name', 'id', 'description', 'begin_from', 'begin_to',
'begin_comment', 'end_from', 'end_to', 'end_comment',
'type_ids'
],
'valid': [],
'invalid': []
}
if class_ == 'place':
columns['allowed'] += ['easting', 'northing']
try:
file_.save(str(file_path))
data_frame = pd.read_csv(file_path, keep_default_na=False)
headers = list(data_frame.columns.values)
if 'name' not in headers: # pragma: no cover
messages['error'].append(_('missing name column'))
raise Exception()
for item in headers: # pragma: no cover
if item not in columns['allowed']:
columns['invalid'].append(item)
del data_frame[item]
if columns['invalid']: # pragma: no cover
messages['warn'].append(
f"{_('invalid columns')}: {','.join(columns['invalid'])}")
headers = list(data_frame.columns.values) # Get clean headers
table_data = []
checked_data = []
origin_ids = []
names = []
missing_name_count = 0
invalid_type_ids = False
invalid_geoms = False
for index, row in data_frame.iterrows():
if not row['name']: # pragma: no cover
missing_name_count += 1
continue
table_row = []
checked_row = {}
for item in headers:
value = row[item]
if item == 'type_ids': # pragma: no cover
type_ids = []
for type_id in value.split():
if Import.check_type_id(type_id, class_):
type_ids.append(type_id)
else:
type_ids.append(
f'<span class="error">{type_id}</span>')
invalid_type_ids = True
value = ' '.join(type_ids)
if item in ['northing', 'easting'] \
and row[item] \
and not is_float(row[item]): # pragma: no cover
value = f'<span class="error">{value}</span>'
invalid_geoms = True # pragma: no cover
if item in [
'begin_from', 'begin_to', 'end_from', 'end_to'
]:
if not value:
value = ''
else:
try:
value = datetime64_to_timestamp(
numpy.datetime64(value))
row[item] = value
except ValueError: # pragma: no cover
row[item] = ''
if str(value) == 'NaT':
value = ''
else:
value = \
f'<span class="error">{value}</span>'
table_row.append(str(value))
checked_row[item] = row[item]
if item == 'name' and form.duplicate.data:
names.append(row['name'].lower())
if item == 'id' and row[item]:
origin_ids.append(str(row['id']))
table_data.append(table_row)
checked_data.append(checked_row)
if invalid_type_ids: # pragma: no cover
messages['warn'].append(_('invalid type ids'))
if invalid_geoms: # pragma: no cover
messages['warn'].append(_('invalid coordinates'))
table = Table(headers, rows=table_data)
# Checking for data inconsistency
if missing_name_count: # pragma: no cover
messages['warn'].append(
f"{_('empty names')}: {missing_name_count}")
doubles = [
item
for item, count in collections.Counter(origin_ids).items()
if count > 1
]
if doubles: # pragma: no cover
messages['error'].append(
f"{_('double IDs in import')}: {', '.join(doubles)}")
existing = Import.get_origin_ids(project, origin_ids) \
if origin_ids else None
if existing:
messages['error'].append(
f"{_('IDs already in database')}: {', '.join(existing)}")
if form.duplicate.data: # Check for possible duplicates
duplicates = Import.check_duplicates(class_, names)
if duplicates: # pragma: no cover
messages['warn'].append(
f"{_('possible duplicates')}: {', '.join(duplicates)}")
if messages['error']:
raise Exception()
except Exception: # pragma: no cover
flash(_('error at import'), 'error')
return render_template(
'import/import_data.html',
form=form,
messages=messages,
file_data=file_data,
title=_('import'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('import'), url_for('import_index')], project,
class_label])
if not form.preview.data and checked_data:
if not file_data['backup_too_old'] or app.config['IS_UNIT_TEST']:
Transaction.begin()
try:
Import.import_data(project, class_, checked_data)
Transaction.commit()
logger.log('info', 'import',
f'import: {len(checked_data)}')
flash(f"{_('import of')}: {len(checked_data)}", 'info')
imported = True
except Exception as e: # pragma: no cover
Transaction.rollback()
logger.log('error', 'import', 'import failed', e)
flash(_('error transaction'), 'error')
return render_template(
'import/import_data.html',
form=form,
file_data=file_data,
table=table,
imported=imported,
messages=messages,
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
[_('import'), url_for('import_index')], project, class_label])
