def show_raw_data(dataset_id, table_name):
if (data_loader.has_access(current_user.username, dataset_id)) is False:
return abort(403)
active_user_handler.make_user_active_in_table(dataset_id, table_name, current_user.username)
start = request.args.get('start')
length = request.args.get('length')
order_column = int(request.args.get('order[0][column]'))
order_direction = request.args.get('order[0][dir]')
raw_table_name = "_raw_" + table_name
ordering = (data_loader.get_column_names(dataset_id, raw_table_name)[order_column], order_direction)
table = data_loader.get_table(dataset_id, raw_table_name, offset=start, limit=length, ordering=ordering)
_table = data_loader.get_table(dataset_id, raw_table_name)
return jsonify(draw=int(request.args.get('draw')),
recordsTotal=len(_table.rows),
recordsFiltered=len(_table.rows),
data=table.rows)
def get_table(dataset_id, table_name):
if (data_loader.has_access(current_user.username, dataset_id)) is False:
return abort(403)
active_user_handler.make_user_active_in_table(dataset_id, table_name, current_user.username)
start = request.args.get('start')
length = request.args.get('length')
order_column_idx = int(request.args.get('order[0][column]'))
order_column_name = request.args.get('columns[{}][data]'.format(order_column_idx))
order_direction = request.args.get('order[0][dir]')
ordering = (order_column_name, order_direction)
search = request.args.get('search[value]')
table = data_loader.get_table(dataset_id, table_name, offset=start, limit=length, ordering=ordering, search=search)
# Make proper data dict out of table rows
data = list()
for r_ix in range(len(table.rows)):
r = dict()
for c_ix in range(len(table.columns)):
r[table.columns[c_ix].name] = table.rows[r_ix][c_ix]
data.append(r)
return jsonify(draw=int(request.args.get('draw')),
recordsTotal=table.total_size,
recordsFiltered=table.total_size,
data=data) # table.rows
def test_impute_missing_data_on_median(self):
# Table
data_loader.create_table('test-table', 0, ['test1', 'test2'])
try:
# Column to int
data_loader.update_column_type(0, 'test-table', 'test2', 'DOUBLE PRECISION')
# Data to test with one missing value
data_loader.insert_row('test-table', 0, ['test1', 'test2'], dict([('test1', '1'), ('test2', 1)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'], dict([('test1', '1'), ('test2', 1)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'], dict([('test1', '1'), ('test2', 6)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'], dict([('test1', '1'), ('test2', 8)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'], dict([('test1', '1')]))
# impute missing data
data_transformer.impute_missing_data_on_average(0, 'test-table', 'test2')
# check imputed data
table = data_loader.get_table(0, 'test-table')
self.assertEqual(len(table.rows), 5)
self.assertEqual(len(table.rows[0]), 3)
self.assertEqual(table.rows[0][2], 1)
self.assertEqual(table.rows[1][2], 1)
self.assertEqual(table.rows[2][2], 6)
self.assertEqual(table.rows[3][2], 8)
self.assertEqual(table.rows[4][2], 4)
finally:
data_loader.delete_table('test-table', 0)
def test_find_and_replace(self):
# Table
data_loader.create_table('test-table', 0, ['test'])
# Data to test replace
data_loader.insert_row('test-table', 0, ['test'], dict([('test', 'appel')]))
data_loader.insert_row('test-table', 0, ['test'], dict([('test', 'citroen')]))
try:
''' Replace strings'''
# Replace ap 'substring' with banaan: appel ==> banaanppel
data_transformer.find_and_replace(0, 'test-table', 'test', 'ap', 'banaan', 'substring')
# Replace citroen 'full replace' with druif: citroen ==> druif
data_transformer.find_and_replace(0, 'test-table', 'test', 'citroen', 'druif', 'full replace')
# check imputed data
table = data_loader.get_table(0, 'test-table')
self.assertEqual(len(table.rows), 2)
self.assertEqual(len(table.rows[0]), 2)
self.assertEqual(table.rows[0][1], 'banaanpel')
self.assertEqual(table.rows[1][1], 'druif')
finally:
data_loader.delete_table('test-table', 0)
def test_find_and_replace_by_regex(self):
# Table
data_loader.create_table('test-table', 0, ['test'])
# Data to test replace
data_loader.insert_row('test-table', 0, ['test'],
dict([('test', 'appel')]))
data_loader.insert_row('test-table', 0, ['test'],
dict([('test', 'citroen')]))
try:
''' Replace string by regex'''
data_transformer.find_and_replace_by_regex(0, 'test-table', 'test',
'a.*', 'banaan')
# check imputed data
table = data_loader.get_table(0, 'test-table')
self.assertEqual(len(table.rows), 2)
self.assertEqual(len(table.rows[0]), 2)
self.assertEqual(table.rows[0][1], 'banaan')
self.assertEqual(table.rows[1][1], 'citroen')
finally:
data_loader.delete_table('test-table', 0)
def get_table(dataset_id, table_name):
if (data_loader.has_access(current_user.username, dataset_id)) is False:
return abort(403)
active_user_handler.make_user_active_in_table(dataset_id, table_name,
current_user.username)
start = request.args.get('start')
length = request.args.get('length')
order_column_idx = int(request.args.get('order[0][column]'))
order_column_name = request.args.get(
'columns[{}][data]'.format(order_column_idx))
order_direction = request.args.get('order[0][dir]')
ordering = (order_column_name, order_direction)
search = request.args.get('search[value]')
table = data_loader.get_table(dataset_id,
table_name,
offset=start,
limit=length,
ordering=ordering,
search=search)
# Make proper data dict out of table rows
data = list()
for r_ix in range(len(table.rows)):
r = dict()
for c_ix in range(len(table.columns)):
r[table.columns[c_ix].name] = table.rows[r_ix][c_ix]
data.append(r)
return jsonify(draw=int(request.args.get('draw')),
recordsTotal=table.total_size,
recordsFiltered=table.total_size,
data=data) # table.rows
def test_find_and_replace(self):
# Table
data_loader.create_table('test-table', 0, ['test'])
# Data to test replace
data_loader.insert_row('test-table', 0, ['test'],
dict([('test', 'appel')]))
data_loader.insert_row('test-table', 0, ['test'],
dict([('test', 'citroen')]))
try:
''' Replace strings'''
# Replace ap 'substring' with banaan: appel ==> banaanppel
data_transformer.find_and_replace(0, 'test-table', 'test', 'ap',
'banaan', 'substring')
# Replace citroen 'full replace' with druif: citroen ==> druif
data_transformer.find_and_replace(0, 'test-table', 'test',
'citroen', 'druif',
'full replace')
# check imputed data
table = data_loader.get_table(0, 'test-table')
self.assertEqual(len(table.rows), 2)
self.assertEqual(len(table.rows[0]), 2)
self.assertEqual(table.rows[0][1], 'banaanpel')
self.assertEqual(table.rows[1][1], 'druif')
finally:
data_loader.delete_table('test-table', 0)
def get_history(dataset_id, table_name):
try:
table = data_loader.get_table(dataset_id, table_name)
return render_template('history/history.html', table=table)
except Exception:
return redirect(url_for('data_service.get_dataset',
dataset_id=dataset_id),
code=303)
def test_get_table(self):
schema_name = 'test-schema'
table_name = 'test-table'
columns = ['test-column']
schema_id = 0
table = Table(table_name, '', columns)
try:
data_loader.create_dataset(schema_name, username)
data_loader.create_table(table_name, schema_id, [])
self.assertEqual(table, data_loader.get_table(schema_id, table_name))
finally:
data_loader.delete_table(table_name, schema_id)
data_loader.delete_dataset(schema_id)
def show_raw_data(dataset_id, table_name):
if (data_loader.has_access(current_user.username, dataset_id)) is False:
return abort(403)
active_user_handler.make_user_active_in_table(dataset_id, table_name,
current_user.username)
start = request.args.get('start')
length = request.args.get('length')
order_column = int(request.args.get('order[0][column]'))
order_direction = request.args.get('order[0][dir]')
raw_table_name = "_raw_" + table_name
ordering = (data_loader.get_column_names(dataset_id,
raw_table_name)[order_column],
order_direction)
table = data_loader.get_table(dataset_id,
raw_table_name,
offset=start,
limit=length,
ordering=ordering)
_table = data_loader.get_table(dataset_id, raw_table_name)
return jsonify(draw=int(request.args.get('draw')),
recordsTotal=len(_table.rows),
recordsFiltered=len(_table.rows),
data=table.rows)
def show_raw_data(dataset_id, table_name):
if (data_loader.has_access(current_user.username, dataset_id)) is False:
return abort(403)
raw_table_name = "_raw_" + table_name
raw_table_exists = data_loader.table_exists(raw_table_name, dataset_id)
if not raw_table_exists:
flash(u"Raw data does not exist.", 'warning')
return redirect(url_for('data_service.get_table', dataset_id=dataset_id, table_name=table_name))
try:
active_user_handler.make_user_active_in_table(dataset_id, table_name, current_user.username)
table = data_loader.get_table(dataset_id, raw_table_name)
title = "Raw data for " + table_name
return render_template('data_service/raw-table-view.html', table=table, title=title)
except Exception:
flash(u"Raw data couldn't be shown.", 'danger')
return redirect(url_for('data_service.get_table', dataset_id=dataset_id, table_name=table_name), code=303)
def get_table(dataset_id, table_name):
if (data_loader.has_access(current_user.username, dataset_id)) is False:
return abort(403)
try:
table = data_loader.get_table(dataset_id, table_name)
statistics = data_loader.get_statistics_for_all_columns(dataset_id, table_name, table.columns)
time_date_transformations = date_time_transformer.get_transformations()
backups = data_loader.get_backups(dataset_id, table_name)
raw_table_name = "_raw_" + table_name
raw_table_exists = data_loader.table_exists(raw_table_name,dataset_id)
current_user.active_schema = dataset_id
active_user_handler.make_user_active_in_table(dataset_id, table_name, current_user.username)
return render_template('data_service/table-view.html', table=table,
time_date_transformations=time_date_transformations,
statistics=statistics, raw_table_exists=raw_table_exists, backups=backups)
except Exception:
flash(u"Table couldn't be shown.", 'danger')
return redirect(url_for('data_service.get_dataset', dataset_id=dataset_id), code=303)
def test_find_and_replace_by_regex(self):
# Table
data_loader.create_table('test-table', 0, ['test'])
# Data to test replace
data_loader.insert_row('test-table', 0, ['test'], dict([('test', 'appel')]))
data_loader.insert_row('test-table', 0, ['test'], dict([('test', 'citroen')]))
try:
''' Replace string by regex'''
data_transformer.find_and_replace_by_regex(0, 'test-table', 'test', 'a.*', 'banaan')
# check imputed data
table = data_loader.get_table(0, 'test-table')
self.assertEqual(len(table.rows), 2)
self.assertEqual(len(table.rows[0]), 2)
self.assertEqual(table.rows[0][1], 'banaan')
self.assertEqual(table.rows[1][1], 'citroen')
finally:
data_loader.delete_table('test-table', 0)
def test_impute_missing_data_on_median(self):
# Table
data_loader.create_table('test-table', 0, ['test1', 'test2'])
try:
# Column to int
data_loader.update_column_type(0, 'test-table', 'test2',
'DOUBLE PRECISION')
# Data to test with one missing value
data_loader.insert_row('test-table', 0, ['test1', 'test2'],
dict([('test1', '1'), ('test2', 1)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'],
dict([('test1', '1'), ('test2', 1)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'],
dict([('test1', '1'), ('test2', 6)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'],
dict([('test1', '1'), ('test2', 8)]))
data_loader.insert_row('test-table', 0, ['test1', 'test2'],
dict([('test1', '1')]))
# impute missing data
data_transformer.impute_missing_data_on_average(
0, 'test-table', 'test2')
# check imputed data
table = data_loader.get_table(0, 'test-table')
self.assertEqual(len(table.rows), 5)
self.assertEqual(len(table.rows[0]), 3)
self.assertEqual(table.rows[0][2], 1)
self.assertEqual(table.rows[1][2], 1)
self.assertEqual(table.rows[2][2], 6)
self.assertEqual(table.rows[3][2], 8)
self.assertEqual(table.rows[4][2], 4)
finally:
data_loader.delete_table('test-table', 0)
def get_history(dataset_id, table_name):
try:
table = data_loader.get_table(dataset_id, table_name)
return render_template('history/history.html', table=table)
except Exception:
return redirect(url_for('data_service.get_dataset', dataset_id=dataset_id), code=303)