def file_index() -> str:
table = Table(['date'] + Table.HEADERS['file'])
file_stats = get_file_stats()
for entity in EntityMapper.get_by_system_type('file', nodes=True):
date = 'N/A'
if entity.id in file_stats:
date = format_date(
datetime.datetime.utcfromtimestamp(
file_stats[entity.id]['date']))
table.rows.append([
date,
link(entity),
entity.print_base_type(),
convert_size(file_stats[entity.id]['size'])
if entity.id in file_stats else 'N/A',
file_stats[entity.id]['ext'] if entity.id in file_stats else 'N/A',
truncate_string(entity.description)
])
if os.name != "posix": # pragma: no cover
# For other operating systems e.g. Windows, we would need adaptions here
return render_template('file/index.html',
table=table,
disk_space_values={})
statvfs = os.statvfs(app.config['UPLOAD_FOLDER_PATH'])
disk_space = statvfs.f_frsize * statvfs.f_blocks
free_space = statvfs.f_frsize * statvfs.f_bavail # Available space without reserved blocks
disk_space_values = {
'total': convert_size(statvfs.f_frsize * statvfs.f_blocks),
'free': convert_size(statvfs.f_frsize * statvfs.f_bavail),
'percent': 100 - math.ceil(free_space / (disk_space / 100))
}
return render_template('file/index.html',
table=table,
disk_space_values=disk_space_values)
def get_disk_space_info() -> Optional[dict[str, Any]]:
if os.name != "posix": # pragma: no cover
return None
statvfs = os.statvfs(app.config['UPLOAD_DIR'])
disk_space = statvfs.f_frsize * statvfs.f_blocks
free_space = statvfs.f_frsize * statvfs.f_bavail
return {
'total': convert_size(statvfs.f_frsize * statvfs.f_blocks),
'free': convert_size(statvfs.f_frsize * statvfs.f_bavail),
'percent': 100 - math.ceil(free_space / (disk_space / 100))}
def export_csv():
path = app.config['EXPORT_FOLDER_PATH'] + '/csv'
writeable = True if os.access(path, os.W_OK) else False
form = ExportCsvForm()
if form.validate_on_submit() and writeable:
Export.export_csv(form)
logger.log('info', 'database', 'CSV export')
flash(_('data was exported as CSV'), 'info')
return redirect(url_for('export_csv'))
table = {'id': 'csv', 'header': ['name', 'size'], 'data': [],
'sort': 'sortList: [[0, 1]],headers: {0: { sorter: "text" }}'}
for file in [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]:
name = basename(file)
if name == '.gitignore':
continue
link = '<a href="{url}">{label}</a>'.format(url=url_for('download_csv', filename=name),
label=uc_first(_('download')))
data = [name, convert_size(os.path.getsize(path + '/' + name)), link]
if is_authorized('admin') and writeable:
confirm = ' onclick="return confirm(\'' + _('Delete %(name)s?', name=name) + '\')"'
delete = '<a href="' + url_for('delete_csv', filename=name)
delete += '" ' + confirm + '>' + uc_first(_('delete')) + '</a>'
data.append(delete)
table['data'].append(data)
return render_template('export/export_csv.html', form=form, table=table, writeable=writeable)
def export_csv() -> str:
path = app.config['EXPORT_FOLDER_PATH'] + '/csv'
writeable = True if os.access(path, os.W_OK) else False
form = ExportCsvForm()
if form.validate_on_submit() and writeable:
Export.export_csv(form)
logger.log('info', 'database', 'CSV export')
flash(_('data was exported as CSV'), 'info')
return redirect(url_for('export_csv'))
table = Table(['name', 'size'], order='[[0, "desc"]]')
for file in [
f for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f))
]:
name = basename(file)
if name == '.gitignore':
continue
link = '<a href="{url}">{label}</a>'.format(url=url_for('download_csv',
filename=name),
label=uc_first(
_('download')))
data = [name, convert_size(os.path.getsize(path + '/' + name)), link]
if is_authorized('admin') and writeable:
confirm = ' onclick="return confirm(\'' + _('Delete %(name)s?',
name=name) + '\')"'
delete = '<a href="' + url_for('delete_csv', filename=name)
delete += '" ' + confirm + '>' + uc_first(_('delete')) + '</a>'
data.append(delete)
table.rows.append(data)
return render_template('export/export_csv.html',
form=form,
table=table,
writeable=writeable)
def export_sql():
path = app.config['EXPORT_FOLDER_PATH'] + '/sql'
writeable = True if os.access(path, os.W_OK) else False
form = ExportSqlForm()
if form.validate_on_submit() and writeable:
if Export.export_sql():
logger.log('info', 'database', 'SQL export')
flash(_('data was exported as SQL'), 'info')
else: # pragma: no cover
logger.log('error', 'database', 'SQL export failed')
flash(_('SQL export failed'), 'error')
return redirect(url_for('export_sql'))
table = {'id': 'sql', 'header': ['name', 'size'], 'data': [],
'sort': 'sortList: [[0, 1]],headers: {0: { sorter: "text" }}'}
for file in [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]:
name = basename(file)
if name == '.gitignore':
continue
url = url_for('download_sql', filename=name)
data = [name, convert_size(os.path.getsize(path + '/' + name)),
'<a href="' + url + '">' + uc_first(_('download')) + '</a>']
if is_authorized('admin') and writeable:
confirm = ' onclick="return confirm(\'' + _('Delete %(name)s?', name=name) + '\')"'
delete = '<a href="' + url_for('delete_sql', filename=name)
delete += '" ' + confirm + '>' + uc_first(_('delete')) + '</a>'
data.append(delete)
table['data'].append(data)
return render_template('export/export_sql.html', form=form, table=table, writeable=writeable)
def get_file_stats(
path: Path = app.config['UPLOAD_DIR']) -> dict[int, dict[str, Any]]:
stats: dict[int, dict[str, Any]] = {}
for file_ in filter(lambda x: x.stem.isdigit(), path.iterdir()):
stats[int(file_.stem)] = {
'ext': file_.suffix,
'size': convert_size(file_.stat().st_size),
'date': file_.stat().st_ctime}
return stats
def file_index():
table = {
'id': 'files',
'header': app.config['TABLE_HEADERS']['file'],
'data': []
}
for file in EntityMapper.get_by_system_type('file'):
table['data'].append(get_base_table_data(file))
statvfs = os.statvfs(app.config['UPLOAD_FOLDER_PATH'])
disk_space = statvfs.f_frsize * statvfs.f_blocks
free_space = statvfs.f_frsize * statvfs.f_bavail # available space without reserved blocks
disk_space_values = {
'total': convert_size(statvfs.f_frsize * statvfs.f_blocks),
'free': convert_size(statvfs.f_frsize * statvfs.f_bavail),
'percent': 100 - math.ceil(free_space / (disk_space / 100))
}
return render_template('file/index.html',
table=table,
disk_space_values=disk_space_values)
def admin_orphans() -> str:
header = ['name', 'class', 'type', 'system type', 'created', 'updated', 'description']
tables = {'orphans': Table(header),
'unlinked': Table(header),
'missing_files': Table(header),
'circular': Table(['entity']),
'nodes': Table(['name', 'root']),
'orphaned_files': Table(['name', 'size', 'date', 'ext'])}
tables['circular'].rows = [[link(entity)] for entity in EntityMapper.get_circular()]
for entity in EntityMapper.get_orphans():
name = 'unlinked' if entity.class_.code in app.config['CODE_CLASS'].keys() else 'orphans'
tables[name].rows.append([link(entity),
link(entity.class_),
entity.print_base_type(),
entity.system_type,
format_date(entity.created),
format_date(entity.modified),
truncate_string(entity.description)])
for node in NodeMapper.get_orphans():
tables['nodes'].rows.append([link(node), link(g.nodes[node.root[-1]])])
# Get orphaned file entities (no corresponding file)
file_ids = []
for entity in EntityMapper.get_by_system_type('file', nodes=True):
file_ids.append(str(entity.id))
if not get_file_path(entity):
tables['missing_files'].rows.append([link(entity),
link(entity.class_),
entity.print_base_type(),
entity.system_type,
format_date(entity.created),
format_date(entity.modified),
truncate_string(entity.description)])
# Get orphaned files (no corresponding entity)
for file in os.scandir(app.config['UPLOAD_FOLDER_PATH']):
name = file.name
if name != '.gitignore' and splitext(file.name)[0] not in file_ids:
confirm = ' onclick="return confirm(\'' + _('Delete %(name)s?', name=name) + '\')"'
tables['orphaned_files'].rows.append([
name,
convert_size(file.stat().st_size),
format_date(datetime.datetime.utcfromtimestamp(file.stat().st_ctime)),
splitext(name)[1],
'<a href="' + url_for('download_file', filename=name) + '">' + uc_first(
_('download')) + '</a>',
'<a href="' + url_for('admin_file_delete', filename=name) + '" ' +
confirm + '>' + uc_first(_('delete')) + '</a>'])
return render_template('admin/orphans.html', tables=tables)
def sql_execute() -> str:
path = app.config['EXPORT_FOLDER_PATH'] + '/sql'
latest_file = None
latest_file_date = None
for file in [
f for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f))
]:
name = basename(file)
if name == '.gitignore':
continue
file_date = datetime.utcfromtimestamp(
os.path.getmtime(path + '/' + file))
if not latest_file_date or file_date > latest_file_date:
latest_file = file
latest_file_date = file_date
file_data = {'backup_to_old': True}
if latest_file:
yesterday = datetime.today() - timedelta(days=1)
file_data['file'] = latest_file
file_data[
'backup_to_old'] = True if yesterday > latest_file_date else False
file_data['size'] = convert_size(
os.path.getsize(path + '/' + latest_file))
file_data['date'] = format_date(latest_file_date)
response = ''
form = SqlForm()
if form.validate_on_submit() and not file_data['backup_to_old']:
g.execute('BEGIN')
try:
g.execute(form.statement.data)
response = '<p>Rows affected: {count}</p>'.format(
count=g.cursor.rowcount)
try:
response += '<p>{rows}</p>'.format(rows=g.cursor.fetchall())
except: # pragma: no cover
pass # Assuming it was no SELECT statement so returning just the rowcount
g.execute('COMMIT')
flash(_('SQL executed'), 'info')
except Exception as e:
g.cursor.execute('ROLLBACK')
logger.log('error', 'database', 'transaction failed', e)
response = e
flash(_('error transaction'), 'error')
return render_template('sql/execute.html',
form=form,
response=response,
file_data=file_data)
def get_table(type_: str, path: Path, writeable: bool) -> Table:
table = Table(['name', 'size'], order=[[0, 'desc']])
for file in [
f for f in path.iterdir()
if (path / f).is_file() and f.name != '.gitignore']:
data = [
file.name,
convert_size(file.stat().st_size),
link(
_('download'),
url_for(f'download_{type_}', filename=file.name))]
if is_authorized('admin') and writeable:
data.append(
delete_link(
file.name,
url_for('delete_export', type_=type_, filename=file.name)))
table.rows.append(data)
return table
def export_sql() -> str:
path = app.config['EXPORT_FOLDER_PATH'] + '/sql'
writeable = True if os.access(path, os.W_OK) else False
form = ExportSqlForm()
if form.validate_on_submit() and writeable:
if Export.export_sql():
logger.log('info', 'database', 'SQL export')
flash(_('data was exported as SQL'), 'info')
else: # pragma: no cover
logger.log('error', 'database', 'SQL export failed')
flash(_('SQL export failed'), 'error')
return redirect(url_for('export_sql'))
table = Table(['name', 'size'], order='[[0, "desc"]]')
for file in [
f for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f))
]:
name = basename(file)
if name == '.gitignore':
continue
url = url_for('download_sql', filename=name)
data = [
name,
convert_size(os.path.getsize(path + '/' + name)),
'<a href="' + url + '">' + uc_first(_('download')) + '</a>'
]
if is_authorized('admin') and writeable:
confirm = ' onclick="return confirm(\'' + _('Delete %(name)s?',
name=name) + '\')"'
delete = '<a href="' + url_for('delete_sql', filename=name)
delete += '" ' + confirm + '>' + uc_first(_('delete')) + '</a>'
data.append(delete)
table.rows.append(data)
return render_template('export/export_sql.html',
form=form,
table=table,
writeable=writeable)
def admin_orphans() -> str:
header = [
'name', 'class', 'type', 'system type', 'created', 'updated',
'description'
]
tabs = {
'orphans':
Tab('orphans', table=Table(header)),
'unlinked':
Tab('unlinked', table=Table(header)),
'types':
Tab('type',
table=Table(
['name', 'root'],
[[link(type_), link(g.types[type_.root[0]])]
for type_ in Type.get_type_orphans()])),
'missing_files':
Tab('missing_files', table=Table(header)),
'orphaned_files':
Tab('orphaned_files', table=Table(['name', 'size', 'date', 'ext'])),
'circular':
Tab('circular_dependencies',
table=Table(['entity'],
[[link(e)]
for e in Entity.get_entities_linked_to_itself()]))
}
for entity in filter(lambda x: not isinstance(x, ReferenceSystem),
Entity.get_orphans()):
tabs['unlinked' if entity.class_.
view else 'orphans'].table.rows.append([
link(entity),
link(entity.class_),
link(entity.standard_type), entity.class_.label,
format_date(entity.created),
format_date(entity.modified), entity.description
])
# Orphaned file entities with no corresponding file
entity_file_ids = []
for entity in Entity.get_by_class('file', types=True):
entity_file_ids.append(entity.id)
if not get_file_path(entity):
tabs['missing_files'].table.rows.append([
link(entity),
link(entity.class_),
link(entity.standard_type), entity.class_.label,
format_date(entity.created),
format_date(entity.modified), entity.description
])
# Orphaned files with no corresponding entity
for file in app.config['UPLOAD_DIR'].iterdir():
if file.name != '.gitignore' \
and os.path.isfile(file) \
and int(file.stem) not in entity_file_ids:
tabs['orphaned_files'].table.rows.append([
file.stem,
convert_size(file.stat().st_size),
format_date(
datetime.datetime.utcfromtimestamp(file.stat().st_ctime)),
file.suffix,
link(_('download'), url_for('download_file',
filename=file.name)),
delete_link(file.name,
url_for('admin_file_delete', filename=file.name))
])
for tab in tabs.values():
tab.buttons = [manual('admin/data_integrity_checks')]
if not tab.table.rows:
tab.content = _('Congratulations, everything looks fine!')
if tabs['orphaned_files'].table.rows and is_authorized('admin'):
text = uc_first(_('delete all files without corresponding entities?'))
tabs['orphaned_files'].buttons.append(
button(_('delete all files'),
url_for('admin_file_delete', filename='all'),
onclick=f"return confirm('{text}')"))
return render_template(
'tabs.html',
tabs=tabs,
title=_('admin'),
crumbs=[[_('admin'), f"{url_for('admin_index')}#tab-data"],
_('orphans')])
def admin_orphans(delete=None):
if delete:
count = EntityMapper.delete_orphans(delete)
flash(_('info orphans deleted:') + ' ' + str(count), 'info')
return redirect(url_for('admin_orphans'))
header = [
'name', 'class', 'type', 'system type', 'created', 'updated',
'description'
]
tables = {
'orphans': {
'id': 'orphans',
'header': header,
'data': []
},
'unlinked': {
'id': 'unlinked',
'header': header,
'data': []
},
'nodes': {
'id': 'nodes',
'header': ['name', 'root'],
'data': []
},
'missing_files': {
'id': 'missing_files',
'header': header,
'data': []
},
'orphaned_files': {
'id': 'orphaned_files',
'data': [],
'header': ['name', 'size', 'date', 'ext']
}
}
for entity in EntityMapper.get_orphans():
name = 'unlinked' if entity.class_.code in app.config[
'CODE_CLASS'].keys() else 'orphans'
tables[name]['data'].append([
link(entity),
link(entity.class_),
entity.print_base_type(), entity.system_type,
format_date(entity.created),
format_date(entity.modified),
truncate_string(entity.description)
])
for node in NodeMapper.get_orphans():
tables['nodes']['data'].append(
[link(node), link(g.nodes[node.root[-1]])])
file_ids = []
# Get orphaned file entities (no corresponding file)
for entity in EntityMapper.get_by_system_type('file'):
file_ids.append(str(entity.id))
if not get_file_path(entity):
tables['missing_files']['data'].append([
link(entity),
link(entity.class_),
entity.print_base_type(), entity.system_type,
format_date(entity.created),
format_date(entity.modified),
truncate_string(entity.description)
])
# Get orphaned files (no corresponding entity)
path = app.config['UPLOAD_FOLDER_PATH']
for file in [
f for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f))
]:
name = basename(file)
file_path = path + '/' + name
if name != '.gitignore' and splitext(name)[0] not in file_ids:
tables['orphaned_files']['data'].append([
name,
convert_size(os.path.getsize(file_path)),
format_date(
datetime.datetime.fromtimestamp(
os.path.getmtime(file_path))),
splitext(name)[1],
'<a href="' + url_for('download_file', filename=name) + '">' +
uc_first(_('download')) + '</a>'
])
return render_template('admin/orphans.html', tables=tables)