def export_shape(dataset_name):
"""Route for /shapes/<shapeset>/ endpoint. Requires a dataset argument
and can apply column specific filters to it.
:param dataset_name: user provided name of target shapeset
:returns: response object result of _export_shape
"""
# Find a way to work these into the validator, they shouldn't be out here.
if dataset_name not in ShapeMetadata.tablenames():
return make_error(dataset_name + ' not found.', 404)
try:
ShapeMetadata.get_by_dataset_name(dataset_name).shape_table
except NoSuchTableError:
return make_error(dataset_name + ' has yet to be ingested.', 404)
meta_params = ('shape', 'data_type', 'location_geom__within', 'job')
request_args = request.args.to_dict()
# Using the 'shape' key triggers the correct validator.
request_args['shape'] = dataset_name
validated_args = validate(
ExportFormatsValidator(only=meta_params),
request_args
)
if validated_args.errors:
return bad_request(validated_args.errors)
elif validated_args.data.get('job'):
return make_job_response('export-shape', validated_args)
else:
query = _export_shape(validated_args)
shapeset = validated_args.data.get('shapeset')
data_type = validated_args.data.get('data_type')
return export_dataset_to_response(shapeset, data_type, query)
def export_shape(dataset_name):
"""Route for /shapes/<shapeset>/ endpoint. Requires a dataset argument
and can apply column specific filters to it.
:param dataset_name: user provided name of target shapeset
:returns: response object result of _export_shape
"""
# Find a way to work these into the validator, they shouldn't be out here.
if dataset_name not in ShapeMetadata.tablenames():
return make_error(dataset_name + ' not found.', 404)
try:
ShapeMetadata.get_by_dataset_name(dataset_name).shape_table
except NoSuchTableError:
return make_error(dataset_name + ' has yet to be ingested.', 404)
meta_params = ('shape', 'data_type', 'location_geom__within', 'job')
request_args = request.args.to_dict()
# Using the 'shape' key triggers the correct validator.
request_args['shape'] = dataset_name
validated_args = validate(ExportFormatsValidator(only=meta_params),
request_args)
if validated_args.errors:
return bad_request(validated_args.errors)
elif validated_args.data.get('job'):
return make_job_response('export-shape', validated_args)
else:
query = _export_shape(validated_args)
shapeset = validated_args.data.get('shapeset')
data_type = validated_args.data.get('data_type')
return export_dataset_to_response(shapeset, data_type, query)
def test_delete_shape(self):
# Can we remove a shape that's fully ingested?
city_meta = postgres_session.query(ShapeMetadata).get(shape_fixtures['city'].table_name)
self.assertIsNotNone(city_meta)
city_meta.remove_table()
postgres_session.commit()
city_meta = postgres_session.query(ShapeMetadata).get(shape_fixtures['city'].table_name)
self.assertIsNone(city_meta)
# Can we remove a shape that's only in the metadata?
dummy_meta = postgres_session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNotNone(dummy_meta)
dummy_meta.remove_table()
postgres_session.commit()
dummy_meta = postgres_session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNone(dummy_meta)
# Add them back to return to original test state
ShapeTests.ingest_fixture(shape_fixtures['city'])
ShapeMetadata.add(human_name='Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False)
postgres_session.commit()
def test_delete_shape(self):
# Can we remove a shape that's fully ingested?
city_meta = postgres_session.query(ShapeMetadata).get(
shape_fixtures['city'].table_name)
self.assertIsNotNone(city_meta)
city_meta.remove_table()
postgres_session.commit()
city_meta = postgres_session.query(ShapeMetadata).get(
shape_fixtures['city'].table_name)
self.assertIsNone(city_meta)
# Can we remove a shape that's only in the metadata?
dummy_meta = postgres_session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNotNone(dummy_meta)
dummy_meta.remove_table()
postgres_session.commit()
dummy_meta = postgres_session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNone(dummy_meta)
# Add them back to return to original test state
ShapeTests.ingest_fixture(shape_fixtures['city'])
ShapeMetadata.add(human_name='Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False)
postgres_session.commit()
def ingest_fixture(fixture):
# Add the fixture to the metadata first
shape_meta = ShapeMetadata.add(caller_session=session, human_name=fixture.human_name, source_url=None)
session.commit()
# Bypass the celery task and call on a ShapeETL directly
ShapeETL(meta=shape_meta, source_path=fixture.path).import_shapefile()
return shape_meta
def _export_dataset_to_response(shapeset, data_type, query=None):
export_format = unicode.lower(unicode(data_type))
# Make a filename that we are reasonably sure to be unique and not occupied by anyone else.
sacrifice_file = tempfile.NamedTemporaryFile()
export_path = sacrifice_file.name
sacrifice_file.close() # Removes file from system.
try:
# Write to that filename.
OgrExport(export_format, export_path, shapeset.name, query).write_file()
# Dump it in the response.
with open(export_path, 'r') as to_export:
resp = make_response(to_export.read(), 200)
extension = _shape_format_to_file_extension(export_format)
# Make the downloaded filename look nice
shapemeta = ShapeMetadata.get_by_dataset_name(shapeset.name)
resp.headers['Content-Type'] = _shape_format_to_content_header(export_format)
resp.headers['Content-Disposition'] = 'attachment; filename={}.{}'.format(shapemeta.human_name, extension)
return resp
except Exception as e:
error_message = 'Failed to export shape dataset {}'.format(shapeset.name)
print repr(e)
return make_response(error_message, 500)
finally:
# Don't leave that file hanging around.
if os.path.isfile(export_path):
os.remove(export_path)
def get_all_shape_datasets():
"""
Fetches metadata for every shape dataset in meta_shape
"""
try:
response_skeleton = {
'meta': {
'status': 'ok',
'message': '',
},
'objects': []
}
geom = request.args.get('location_geom__within')
if geom:
geom = make_sql_ready_geom(geom)
public_listing = ShapeMetadata.index(geom)
response_skeleton['objects'] = public_listing
status_code = 200
except Exception as e:
print e.message
response_skeleton = {
'meta': {
'status': 'error',
'message': '',
}
}
status_code = 500
resp = make_response(json.dumps(response_skeleton), status_code)
resp.headers['Content-Type'] = 'application/json'
return resp
def get_all_shape_datasets():
"""
Fetches metadata for every shape dataset in meta_shape
"""
try:
response_skeleton = {
'meta': {
'status': 'ok',
'message': '',
},
'objects': []
}
geom = request.args.get('location_geom_within')
if geom:
geom = make_sql_ready_geom(geom)
public_listing = ShapeMetadata.index(geom)
response_skeleton['objects'] = public_listing
status_code = 200
except Exception as e:
print e.message
response_skeleton = {
'meta': {
'status': 'error',
'message': '',
}
}
status_code = 500
resp = make_response(json.dumps(response_skeleton), status_code)
resp.headers['Content-Type'] = 'application/json'
return resp
def get_all_shape_datasets():
"""
Fetches metadata for every shape dataset in meta_shape
"""
try:
response_skeleton = {
'meta': {
'status': 'ok',
'message': '',
},
'objects': []
}
public_listing = ShapeMetadata.index(caller_session=session)
response_skeleton['objects'] = public_listing
status_code = 200
except Exception as e:
print e.message
response_skeleton = {
'meta': {
'status': 'error',
'message': '',
}
}
status_code = 500
resp = make_response(json.dumps(response_skeleton), status_code)
resp.headers['Content-Type'] = 'application/json'
return resp
def ingest_fixture(fixture):
# Add the fixture to the metadata first
shape_meta = ShapeMetadata.add(human_name=fixture.human_name,
source_url=None,
update_freq=fixture.update_freq,
approved_status=False)
session.commit()
# Bypass the celery task and call on a ShapeETL directly
ShapeETL(meta=shape_meta, source_path=fixture.path).add()
return shape_meta
def ingest_fixture(fixture):
# Add the fixture to the metadata first
shape_meta = ShapeMetadata.add(human_name=fixture.human_name,
source_url=None,
update_freq=fixture.update_freq,
approved_status=False)
session.commit()
# Bypass the celery task and call on a ShapeETL directly
ShapeETL(meta=shape_meta, source_path=fixture.path).add()
return shape_meta
def view_datasets():
datasets_pending = fetch_pending_tables(MetaTable)
shapes_pending = fetch_pending_tables(ShapeMetadata)
datasets = MetaTable.get_all_with_etl_status()
shapesets = ShapeMetadata.get_all_with_etl_status()
return render_template('admin/view-datasets.html',
datasets_pending=datasets_pending,
shapes_pending=shapes_pending,
datasets=datasets,
shape_datasets=shapesets)
def view_datasets():
datasets_pending = fetch_pending_tables(MetaTable)
shapes_pending = fetch_pending_tables(ShapeMetadata)
datasets = MetaTable.get_all_with_etl_status()
shapesets = ShapeMetadata.get_all_with_etl_status()
return render_template('admin/view-datasets.html',
datasets_pending=datasets_pending,
shapes_pending=shapes_pending,
datasets=datasets,
shape_datasets=shapesets)
def test_delete_shape(self):
# Can we remove a shape that's fully ingested?
city_meta = session.query(ShapeMetadata).get(fixtures["city"].table_name)
self.assertIsNotNone(city_meta)
city_meta.remove_table(caller_session=session)
session.commit()
city_meta = session.query(ShapeMetadata).get(fixtures["city"].table_name)
self.assertIsNone(city_meta)
# Can we remove a shape that's only in the metadata?
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNotNone(dummy_meta)
dummy_meta.remove_table(caller_session=session)
session.commit()
dummy_meta = session.query(ShapeMetadata).get(self.dummy_name)
self.assertIsNone(dummy_meta)
# Add them back to return to original test state
ShapeTests.ingest_fixture(fixtures["city"])
ShapeMetadata.add(caller_session=session, human_name=u"Dummy Name", source_url=None)
session.commit()
def get_all_shape_datasets():
"""Fetches metadata for every shape dataset in meta_shape.
"""
try:
response_skeleton = {
'meta': {
'status': 'ok',
'message': '',
},
'objects': []
}
geom = request.args.get('location_geom__within')
simple_bbox = request.args.get('simple_bbox')
if geom:
geom = make_fragment_str(
extract_first_geometry_fragment(geom)
)
if simple_bbox:
public_listing = ShapeMetadata.simple_index(geom)
else:
public_listing = ShapeMetadata.index(geom)
response_skeleton['objects'] = public_listing
status_code = 200
except Exception as e:
response_skeleton = {
'meta': {
'status': 'error',
'message': str(e),
}
}
status_code = 500
resp = make_response(json.dumps(response_skeleton, default=str), status_code)
resp.headers['Content-Type'] = 'application/json'
return resp
def get_all_shape_datasets():
"""Fetches metadata for every shape dataset in meta_shape.
"""
try:
response_skeleton = {
'meta': {
'status': 'ok',
'message': '',
},
'objects': []
}
geom = request.args.get('location_geom__within')
simple_bbox = request.args.get('simple_bbox')
if geom:
geom = make_fragment_str(extract_first_geometry_fragment(geom))
if simple_bbox:
public_listing = ShapeMetadata.simple_index(geom)
else:
public_listing = ShapeMetadata.index(geom)
response_skeleton['objects'] = public_listing
status_code = 200
except Exception as e:
response_skeleton = {
'meta': {
'status': 'error',
'message': str(e),
}
}
status_code = 500
resp = make_response(json.dumps(response_skeleton, default=str),
status_code)
resp.headers['Content-Type'] = 'application/json'
return resp
def shape_meta_from_submit_form(form, is_approved):
md = ShapeMetadata.add(
human_name=form['dataset_name'],
source_url=form['file_url'],
view_url=form.get('view_url'),
attribution=form.get('dataset_attribution'),
description=form.get('dataset_description'),
update_freq=form['update_frequency'],
contributor_name=form['contributor_name'],
contributor_organization=form.get('contributor_organization'),
contributor_email=form['contributor_email'],
approved_status=is_approved)
postgres_session.commit()
return md
def shape_meta_from_submit_form(form, is_approved):
md = ShapeMetadata.add(
human_name=form['dataset_name'],
source_url=form['file_url'],
view_url=form.get('view_url'),
attribution=form.get('dataset_attribution'),
description=form.get('dataset_description'),
update_freq=form['update_frequency'],
contributor_name=form['contributor_name'],
contributor_organization=form.get('contributor_organization'),
contributor_email=form['contributor_email'],
approved_status=is_approved)
postgres_session.commit()
return md
def ingest_shapes(cls):
fixtures = [f for k, f in shape_fixtures.items() if k != 'changed_neighborhoods']
fixture_table_names = [f.table_name for f in fixtures]
drop_tables(fixture_table_names)
postgres_session.commit()
for fixture in fixtures:
cls.ingest_fixture(fixture)
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(human_name='Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False).dataset_name
postgres_session.commit()
def setUpClass(cls, shutdown=False):
# Remove tables that we're about to recreate.
# This doesn't happen in teardown because I find it helpful
# to inspect them in the DB after running the tests.
meta_table_names = ['meta_shape']
fixture_table_names = [fixture.table_name for key, fixture in fixtures.iteritems()]
drop_tables(meta_table_names + fixture_table_names)
# Re-add meta tables
init_meta()
# Fully ingest the fixtures
BasePlenarioTest.ingest_fixture(fixtures['city'])
BasePlenarioTest.ingest_fixture(fixtures['streets'])
BasePlenarioTest.ingest_fixture(fixtures['zips'])
BasePlenarioTest.ingest_fixture(fixtures['neighborhoods'])
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(human_name=u'Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False).dataset_name
session.commit()
tables_to_drop = [
'flu_shot_clinics',
'landmarks',
'crimes',
'meta_master'
]
drop_tables(tables_to_drop)
init_meta()
ingest_from_fixture(flu_shot_meta, flu_path)
ingest_from_fixture(landmarks_meta, landmarks_path)
ingest_from_fixture(crime_meta, crime_path)
cls.app = create_app().test_client()
'''/detail'''
def setUpClass(cls, shutdown=False):
# Remove tables that we're about to recreate.
# This doesn't happen in teardown because I find it helpful
# to inspect them in the DB after running the tests.
meta_table_names = ['meta_shape']
fixture_table_names = [
fixture.table_name for key, fixture in fixtures.iteritems()
]
drop_tables(meta_table_names + fixture_table_names)
# Re-add meta tables
init_meta()
# Fully ingest the fixtures
BasePlenarioTest.ingest_fixture(fixtures['city'])
BasePlenarioTest.ingest_fixture(fixtures['streets'])
BasePlenarioTest.ingest_fixture(fixtures['zips'])
BasePlenarioTest.ingest_fixture(fixtures['neighborhoods'])
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(human_name=u'Dummy Name',
source_url=None,
update_freq='yearly',
approved_status=False).dataset_name
session.commit()
tables_to_drop = [
'flu_shot_clinics', 'landmarks', 'crimes', 'meta_master'
]
drop_tables(tables_to_drop)
init_meta()
ingest_from_fixture(flu_shot_meta, flu_path)
ingest_from_fixture(landmarks_meta, landmarks_path)
ingest_from_fixture(crime_meta, crime_path)
cls.app = create_app().test_client()
'''/detail'''
def view_datasets():
datasets_pending = session.query(MetaTable).\
filter(MetaTable.approved_status != True).\
all()
shapes_pending = session.query(ShapeMetadata).\
filter(ShapeMetadata.approved_status != True).\
all()
try:
q = text('''
SELECT m.*, c.status, c.task_id
FROM meta_master AS m
LEFT JOIN celery_taskmeta AS c
ON c.id = (
SELECT id FROM celery_taskmeta
WHERE task_id = ANY(m.result_ids)
ORDER BY date_done DESC
LIMIT 1
)
WHERE m.approved_status = 'true'
''')
with engine.begin() as c:
datasets = list(c.execute(q))
except NoSuchTableError:
datasets = session.query(MetaTable)\
.filter(MetaTable.approved_status == True)\
.all()
try:
shape_datasets = ShapeMetadata.get_all_with_etl_status()
except NoSuchTableError:
# If we can't find shape metadata, soldier on.
shape_datasets = None
return render_template('admin/view-datasets.html',
datasets_pending=datasets_pending,
shapes_pending=shapes_pending,
datasets=datasets,
shape_datasets=shape_datasets)
def view_datasets():
datasets_pending = session.query(MetaTable).\
filter(MetaTable.approved_status != True).\
all()
shapes_pending = session.query(ShapeMetadata).\
filter(ShapeMetadata.approved_status != True).\
all()
try:
q = text('''
SELECT m.*, c.status, c.task_id
FROM meta_master AS m
LEFT JOIN celery_taskmeta AS c
ON c.id = (
SELECT id FROM celery_taskmeta
WHERE task_id = ANY(m.result_ids)
ORDER BY date_done DESC
LIMIT 1
)
WHERE m.approved_status = 'true'
''')
with engine.begin() as c:
datasets = list(c.execute(q))
except NoSuchTableError:
datasets = session.query(MetaTable)\
.filter(MetaTable.approved_status == True)\
.all()
try:
shape_datasets = ShapeMetadata.get_all_with_etl_status()
except NoSuchTableError:
# If we can't find shape metadata, soldier on.
shape_datasets = None
return render_template('admin/view-datasets.html',
datasets_pending=datasets_pending,
shapes_pending=shapes_pending,
datasets=datasets,
shape_datasets=shape_datasets)
def setUpClass(cls):
# Remove tables that we're about to recreate.
# This doesn't happen in teardown because I find it helpful to inspect them in the DB after running the tests.
meta_table_names = ["dat_master", "meta_shape", "meta_master", "plenario_user"]
fixture_table_names = [fixture.table_name for key, fixture in fixtures.iteritems()]
drop_tables(meta_table_names + fixture_table_names)
# Re-add meta tables
init_master_meta_user()
# Fully ingest the fixtures
ShapeTests.ingest_fixture(fixtures["city"])
ShapeTests.ingest_fixture(fixtures["streets"])
ShapeTests.ingest_fixture(fixtures["zips"])
# Add a dummy dataset to the metadata without ingesting a shapefile for it
cls.dummy_name = ShapeMetadata.add(
caller_session=session, human_name=u"Dummy Name", source_url=None
).dataset_name
session.commit()
cls.app = create_app().test_client()
def export_dataset_to_response(shapeset, data_type, query=None):
export_format = str.lower(str(data_type))
# Make a filename that we are reasonably sure to be unique and not occupied by anyone else.
sacrifice_file = tempfile.NamedTemporaryFile()
export_path = sacrifice_file.name
sacrifice_file.close() # Removes file from system.
try:
# Write to that filename.
OgrExport(export_format, export_path, shapeset.name,
query).write_file()
# Dump it in the response.
with open(export_path, 'rb') as to_export:
resp = make_response(to_export.read(), 200)
extension = _shape_format_to_file_extension(export_format)
# Make the downloaded filename look nice
shapemeta = ShapeMetadata.get_by_dataset_name(shapeset.name)
resp.headers['Content-Type'] = _shape_format_to_content_header(
export_format)
resp.headers[
'Content-Disposition'] = "attachment; filename='{}.{}'".format(
shapemeta.human_name, extension)
return resp
except Exception as e:
error_message = 'Failed to export shape dataset {}'.format(
shapeset.name)
print((repr(e)))
return make_response(error_message, 500)
finally:
# Don't leave that file hanging around.
if os.path.isfile(export_path):
os.remove(export_path)
def __init__(self, human_name, file_name):
self.human_name = human_name
self.table_name = ShapeMetadata.make_table_name(human_name)
self.path = os.path.join(FIXTURE_PATH, file_name)
self.update_freq = 'yearly'
def shape_status():
table_name = request.args['dataset_name']
shape_meta = ShapeMetadata.get_metadata_with_etl_result(table_name)
return render_template('admin/shape-status.html', shape=shape_meta)
def shape_already_submitted(name):
shape = ShapeMetadata.get_by_human_name(name)
print shape, shape is not None
return shape is not None
# Many methods in response.py rely on information that used to be provided
# by the old ParamValidator attributes. This namedtuple carries that same
# info around, and allows me to not have to rewrite any response code.
ValidatorResult = namedtuple('ValidatorResult', 'data errors warnings')
# converters
# ==========
# Callables which are used to convert request arguments to their correct types.
converters = {
'agg': str,
'buffer': int,
'dataset': lambda x: MetaTable.get_by_dataset_name(x).point_table,
'shapeset': lambda x: ShapeMetadata.get_by_dataset_name(x).shape_table,
'data_type': str,
'shape': lambda x: ShapeMetadata.get_by_dataset_name(x).shape_table,
'dataset_name__in': lambda x: x.split(','),
'date__time_of_day_ge': int,
'date__time_of_day_le': int,
'obs_date__ge': lambda x: parser.parse(x).date(),
'obs_date__le': lambda x: parser.parse(x).date(),
'date': lambda x: parser.parse(x).date(),
'point_date': lambda x: parser.parse(x),
'offset': int,
'resolution': int,
'geom': lambda x: make_fragment_str(extract_first_geometry_fragment(x)),
}
def __init__(self, human_name, file_name):
self.human_name = human_name
self.table_name = ShapeMetadata.make_table_name(human_name)
self.path = os.path.join(FIXTURE_PATH, file_name)
self.update_freq = 'yearly'
def validate(validator, request_args):
"""Validate a dictionary of arguments. Substitute all missing fields with
defaults if not explicitly told to do otherwise.
:param validator: what kind of validator to use
:param request_args: dictionary of arguments from a request object
:returns: ValidatorResult namedtuple"""
args = request_args.copy()
# For validator dataset_name__in... need to find a better way to
# make it play nice with the validator.
if args.get('dataset_name__in'):
args['dataset_name__in'] = args['dataset_name__in'].split(',')
# This first validation step covers conditions that are dataset
# agnostic. These are values can be used to apply to all datasets
# (ex. obs_date), or concern the format of the response (ex. limit,
# datatype, offset).
# If there are errors, fail quickly and return.
result = validator.load(args)
if result.errors:
return result
# If all arguments are valid, fill in validator defaults.
result = validator.dump(result.data)
# Certain values will be dumped as strings. This conversion
# makes them into their corresponding type. (ex. Table)
convert(result.data)
# Holds messages concerning unnecessary parameters. These can be either
# junk parameters, or redundant column parameters if a tree filter was
# used.
warnings = []
# At this point validation splits. We can either validate tree-style column
# arguments or validate them individually. We don't do both.
# Determine unchecked parameters provided in the request.
unchecked = set(args.keys()) - set(validator.fields.keys())
# If tree filters were provided, ignore ALL unchecked parameters that are
# not tree filters or response format information.
if has_tree_filters(request_args):
for key in request_args:
value = args[key]
if 'filter' in key:
# This pattern matches the last occurrence of the '__' pattern.
# Prevents an error that is caused by dataset names with trailing
# underscores.
t_name = re.split(r'__(?!_)', key)[0]
# Report a filter which specifies a non-existent tree.
try:
table = MetaTable.get_by_dataset_name(t_name).point_table
except (AttributeError, NoSuchTableError):
try:
table = ShapeMetadata.get_by_dataset_name(t_name).shape_table
except (AttributeError, NoSuchTableError):
result.errors[t_name] = "Table name {} could not be found.".format(t_name)
return result
# Report a tree which causes the JSON parser to fail.
# Or a tree whose value is not valid.
try:
cond_tree = json.loads(value)
if valid_tree(table, cond_tree):
result.data[key] = cond_tree
except (ValueError, KeyError) as err:
result.errors[t_name] = "Bad tree: {} -- causes error {}.".format(value, err)
return result
# These keys just have to do with the formatting of the JSON response.
# We keep these values around even if they have no effect on a condition
# tree.
elif key in {'geom', 'offset', 'limit', 'agg', 'obs_date__le', 'obs_date__ge'}:
pass
# These keys are also ones that should be passed over when searching for
# unused params. They are used, just in different forms later on, so no need
# to report them.
elif key in {'shape', 'dataset_name', 'dataset_name__in'}:
pass
# If the key is not a filter, and not used to format JSON, report
# that we ignored it.
else:
warnings.append("Unused parameter {}, you cannot specify both "
"column and filter arguments.".format(key))
# If no tree filters were provided, see if any of the unchecked parameters
# are usable as column conditions.
else:
try:
table = result.data['dataset']
except KeyError:
table = result.data.get('shapeset')
for param in unchecked:
field = param.split('__')[0]
if table is not None:
try:
valid_column_condition(table, field, args[param])
result.data[param] = args[param]
except KeyError:
warnings.append('Unused parameter value "{}={}"'.format(param, args[param]))
warnings.append('{} is not a valid column for {}'.format(param, table))
except ValueError:
warnings.append('Unused parameter value "{}={}"'.format(param, args[param]))
warnings.append('{} is not a valid value for {}'.format(args[param], param))
# ValidatorResult(dict, dict, list)
return ValidatorResult(result.data, result.errors, warnings)
def shape_already_submitted(name):
shape = ShapeMetadata.get_by_human_name(name)
return shape is not None
def shape_status():
table_name = request.args['dataset_name']
shape_meta = ShapeMetadata.get_metadata_with_etl_result(table_name)
return render_template('admin/shape-status.html', shape=shape_meta)