def test_optional_argument():
assert parse_url_path('foo/bar/:changes') == {
'path': 'foo/bar',
'changes': None
}
assert parse_url_path('foo/bar/:changes/42') == {
'path': 'foo/bar',
'changes': 42
}
assert parse_url_path('foo/bar/:changes/-42') == {
'path': 'foo/bar',
'changes': -42
}
assert build_url_path({
'path': 'foo/bar',
'changes': None
}) == 'foo/bar/:changes'
assert build_url_path({
'path': 'foo/bar',
'changes': 42
}) == 'foo/bar/:changes/42'
assert build_url_path({
'path': 'foo/bar',
'changes': -42
}) == 'foo/bar/:changes/-42'
def get_current_location(path, params):
parts = path.split('/') if path else []
loc = [('root', '/')]
if 'source' in params:
if 'id' in params:
if 'changes' in params:
loc += (
[(p, '/' + '/'.join(parts[:i]))
for i, p in enumerate(parts, 1)] +
[(':source/' + params['source'],
'/' + '/'.join(parts + [':source', params['source']]))] +
[(params['id']['value'][:8],
'/' + build_url_path({
'path': path,
'id': params['id'],
'source': params['source'],
}))] + [(':changes', None)])
else:
loc += ([(p, '/' + '/'.join(parts[:i]))
for i, p in enumerate(parts, 1)] +
[(':source/' + params['source'], '/' +
'/'.join(parts + [':source', params['source']]))] +
[(params['id']['value'][:8],
None)] + [(':changes',
'/' + build_url_path(
{
'path': path,
'id': params['id'],
'source': params['source'],
'changes': None,
}))])
else:
if 'changes' in params:
loc += ([(p, '/' + '/'.join(parts[:i]))
for i, p in enumerate(parts, 1)] +
[(':source/' + params['source'],
'/' + build_url_path({
'path': path,
'source': params['source'],
}))] + [(':changes', None)])
else:
loc += ([(p, '/' + '/'.join(parts[:i]))
for i, p in enumerate(parts, 1)] +
[(':source/' + params['source'], None)] +
[(':changes',
'/' + build_url_path({
'path': path,
'source': params['source'],
'changes': None,
}))])
else:
loc += ([(p, '/' + '/'.join(parts[:i]))
for i, p in enumerate(parts[:-1], 1)] + [(p, None)
for p in parts[-1:]])
return loc
def get_cell(params, prop, value, shorten=False, color=None):
COLORS = {
'change': '#B2E2AD',
'null': '#C1C1C1',
}
link = None
if prop.name == 'id' and value:
extra = {}
if 'source' in params:
extra['source'] = params['source']
link = '/' + build_url_path({
'path': params['path'],
'id': {
'value': value,
'type': None
},
**extra,
})
if shorten:
value = value[:8]
elif hasattr(prop, 'ref') and prop.ref and value:
extra = {}
if 'source' in params:
extra['source'] = params['source']
link = '/' + build_url_path({
'path': prop.ref,
'id': {
'value': value,
'type': None
},
**extra,
})
if shorten:
value = value[:8]
if isinstance(value, datetime.datetime):
value = value.isoformat()
max_column_length = 200
if shorten and isinstance(value, str) and len(value) > max_column_length:
value = value[:max_column_length] + '...'
if value is None:
value = ''
if color is None:
color = 'null'
return {
'value': value,
'link': link,
'color': COLORS[color] if color else None,
}
def test_build_url_path():
assert build_url_path({
'path': 'foo/bar',
'id': {
'value': '42',
'type': 'sha1'
}
}) == 'foo/bar/42'
assert build_url_path({
'path': 'foo/bar',
'id': {
'value': '42',
'type': 'sha1'
},
'source': 'gov/org'
}) == 'foo/bar/42/:source/gov/org'
def test_no_args():
string = 'foo/bar/:count'
params = {
'path': 'foo/bar',
'count': [],
}
assert parse_url_path(string) == params
assert build_url_path(params) == string
def test_format():
string = 'foo/bar/:format/csv'
params = {
'path': 'foo/bar',
'format': 'csv',
}
assert parse_url_path(string) == params
assert build_url_path(params) == string
def test_id_integer():
string = 'foo/bar/42'
params = {
'path': 'foo/bar',
'id': {'value': 42, 'type': 'integer'},
}
assert parse_url_path(string) == params
assert build_url_path(params) == string
def get_cell(params, prop, value, shorten=False, color=None):
link = None
if prop.name == 'id' and value:
link = '/' + build_url_path({
'path': params['path'],
'id': {
'value': value,
'type': None
},
'source': params['source'],
})
if shorten:
value = value[:8]
elif prop.ref and value:
link = '/' + build_url_path({
'path': prop.ref,
'id': {
'value': value,
'type': None
},
'source': params['source'],
})
if shorten:
value = value[:8]
if isinstance(value, datetime.datetime):
value = value.isoformat()
max_column_length = 200
if shorten and isinstance(value, str) and len(value) > max_column_length:
value = value[:max_column_length] + '...'
if value is None:
value = ''
if color is None:
color = 'null'
return {
'value': value,
'link': link,
'color': COLORS[color] if color else None,
}
def test_id_sha1():
string = 'foo/bar/69a33b149af7a7eeb25026c8cdc09187477ffe21'
params = {
'path': 'foo/bar',
'id': {
'value': '69a33b149af7a7eeb25026c8cdc09187477ffe21',
'type': 'sha1',
},
}
assert parse_url_path(string) == params
assert build_url_path(params) == string
def test_sort():
string = 'foo/bar/:sort/a/-b'
params = {
'path': 'foo/bar',
'sort': [
{'name': 'a', 'ascending': True},
{'name': 'b', 'ascending': False},
],
}
assert parse_url_path(string) == params
assert build_url_path(params) == string
async def create_http_response(context, params, request):
config = context.get('config')
_params = params.params
path = params.model
store = context.get('store')
manifest = store.manifests['default']
if params.format == 'html':
header = []
data = []
formats = []
items = []
datasets = []
row = []
context.bind('transaction', manifest.backend.transaction)
try:
model = get_model_from_params(manifest, params.model, _params)
except NotFound:
model = None
loc = get_current_location(model, path, _params)
if model:
formats = [
('CSV', '/' + build_url_path({
**_params, 'format': 'csv'
})),
('JSON', '/' + build_url_path({
**_params, 'format': 'json'
})),
('JSONL', '/' + build_url_path({
**_params, 'format': 'jsonl'
})),
('ASCII', '/' + build_url_path({
**_params, 'format': 'ascii'
})),
]
if params.changes:
data = get_changes(context, _params)
header = next(data)
data = list(reversed(list(data)))
else:
if params.id:
row = list(get_row(context, _params))
else:
data = get_data(context, _params)
header = next(data)
data = list(data)
else:
datasets_by_object = get_datasets_by_object(context)
tree = build_path_tree(
manifest.objects.get('model', {}).keys(),
datasets_by_object.keys(),
)
items = get_directory_content(tree, path) if path in tree else []
datasets = list(get_directory_datasets(datasets_by_object, path))
templates = Jinja2Templates(
directory=pres.resource_filename('spinta', 'templates'))
return templates.TemplateResponse(
'base.html', {
'request': request,
'location': loc,
'formats': formats,
'items': items,
'datasets': datasets,
'header': header,
'data': data,
'row': row,
})
elif params.format in config.exporters:
if not params.model:
raise HTTPException(status_code=404)
try:
model = get_model_from_params(manifest, params.model, _params)
except NotFound as e:
raise HTTPException(status_code=404, detail=str(e))
if request.method == 'POST':
context.bind('transaction',
manifest.backend.transaction,
write=True)
# only `application/json` is supported
ct = request.headers.get('content-type')
if ct != 'application/json':
raise HTTPException(
status_code=415,
detail="only 'application/json' content-type is supported",
)
# make sure json is valid
try:
data = await request.json()
except json.decoder.JSONDecodeError:
raise HTTPException(
status_code=400,
detail="not a valid json",
)
if 'revision' in data.keys():
raise HTTPException(
status_code=400,
detail="cannot create 'revision'",
)
data = commands.push(context,
model,
model.backend,
data,
action='insert')
return JSONResponse(data, status_code=201)
context.set('transaction', manifest.backend.transaction())
if params.changes:
rows = commands.changes(
context,
model,
model.backend,
id=_params.get('id', {}).get('value'),
limit=_params.get('limit', 100),
offset=_params.get('changes', -10),
)
# TODO: see below (look for peek_and_stream)
rows = peek_and_stream(rows)
elif params.id:
rows = [
commands.get(context, model, model.backend,
_params['id']['value'])
]
_params['wrap'] = False
else:
rows = commands.getall(
context,
model,
model.backend,
show=_params.get('show'),
sort=_params.get('sort', [{
'name': 'id',
'ascending': True
}]),
offset=_params.get('offset'),
limit=_params.get('limit', 100),
count='count' in _params,
)
# TODO: Currently authorization and many other thins happens inside
# backend functions and on iterator. If anything failes, we
# get a RuntimeError, because error happens after response is
# started, that means we no longer can return correct HTTP
# response with an error. That is why, we first run one
# iteration before returning StreamingResponse, to catch
# possible errors.
#
# In other words, all possible checks should happend before
# starting response.
#
# Current solution with PeekStream is just a temporary
# workaround.
rows = peek_and_stream(rows)
exporter = config.exporters[params.format]
media_type = exporter.content_type
_params = {k: v for k, v in _params.items() if k in exporter.params}
stream = aiter(exporter(rows, **_params))
return StreamingResponse(stream, media_type=media_type)
else:
raise HTTPException(status_code=404)
async def homepage(request):
global store
url_path = request.path_params['path'].strip('/')
params = parse_url_path(url_path)
path = params['path']
fmt = params.get('format', 'html')
if fmt == 'html':
header = []
data = []
formats = []
items = []
datasets = []
row = []
loc = get_current_location(path, params)
if 'source' in params:
formats = [
('CSV', '/' + build_url_path({
**params, 'format': 'csv'
})),
('JSON', '/' + build_url_path({
**params, 'format': 'json'
})),
('JSONL', '/' + build_url_path({
**params, 'format': 'jsonl'
})),
('ASCII', '/' + build_url_path({
**params, 'format': 'asciitable'
})),
]
if 'changes' in params:
data = get_changes(store, params)
header = next(data)
data = list(reversed(list(data)))
else:
if 'id' in params:
row = list(get_row(store, params))
else:
data = get_data(store, params)
header = next(data)
data = list(data)
else:
datasets_by_object = get_datasets_by_object(store)
tree = build_path_tree(
store.objects['default'].get('model', {}).keys(),
datasets_by_object.keys(),
)
items = get_directory_content(tree, path) if path in tree else []
datasets = list(get_directory_datasets(datasets_by_object, path))
return templates.TemplateResponse(
'base.html', {
'request': request,
'location': loc,
'formats': formats,
'items': items,
'datasets': datasets,
'header': header,
'data': data,
'row': row,
})
elif fmt in ('csv', 'json', 'jsonl', 'asciitable'):
async def generator(rows, fmt, params):
for data in store.export(rows, fmt, params):
yield data
if 'changes' in params:
rows = store.changes(params)
elif 'id' in params:
rows = [store.get(params['path'], params['id']['value'], params)]
params['wrap'] = False
else:
rows = store.getall(params['path'], params)
media_types = {
'csv': 'text/csv',
'json': 'application/json',
'jsonl': 'application/x-json-stream',
'asciitable': 'text/plain',
}
if fmt == 'asciitable':
params['column_width'] = 42
return StreamingResponse(generator(rows, fmt, params),
media_type=media_types[fmt])
else:
raise HTTPException(status_code=404)