def put(self, key):
"""
A REST endpoint for adding or editing a single layer.
All registration requests must contain entries for all languages and will be validated against a JSON schema.
:param smallkey: A unique identifier for the dataset (can be any unique string, but preferably should be short)
:type smallkey: str
:returns: JSON Response -- 201 on success; 400 with JSON payload of an errors array on failure
"""
try:
req = json.loads(request.data)
except Exception as e:
current_app.logger.error(e.message)
return '{"errors":["Unparsable json"]}', 400
errors = get_registration_errors(req)
if errors:
resp = {'errors': errors}
current_app.logger.info(resp)
return Response(json.dumps(resp), mimetype='application/json', status=400)
try:
v2_node, v1_node = regparse.make_node(key, req, current_app.config)
except regparse.metadata.MetadataException as mde:
current_app.logger.warning('Metadata could not be retrieved for layer', exc_info=mde)
abort(400, msg=mde.message)
except regparse.ServiceEndpointException as se:
current_app.logger.warning('Problem reading service endpoints', exc_info=se)
abort(400, msg=se.message)
current_app.logger.debug(v2_node)
current_app.logger.debug(v1_node)
db.put_doc(key, v2_node.values()[0]['layerType'], req, layer_config=v2_node, v1_config=v1_node)
current_app.logger.info('added a key %s' % key)
return Response(json.dumps(dict(key=key)), mimetype='application/json', status=201)
def post(self, key):
"""
A REST endpoint for upgrading a previous registration from v1 to v2.
:param key: A unique identifier for the dataset
:type key: str
:returns: JSON Response -- 200 on success; 4xx if problems are encountered
"""
dbdata = db.get_raw(key)
if dbdata is None:
return '{"msg":"Record not found in database"}', 404
elif dbdata.get('version') == '2.0':
return '{"msg":"Already upgraded"}', 200
elif dbdata['data'].get('request') is None:
return '{"msg":"Previous request was not cached (request caching added in 1.8.0)"}', 409
try:
v1_request = dbdata['data']['request']
upgrade_method = wms_upgrade if v1_request['payload_type'] == 'wms' else feat_upgrade
v2_request = {lang: upgrade_method(v1_request[lang]) for lang in current_app.config['LANGS']}
print v2_request
v2_node, v1_node = regparse.make_node(key, v2_request, current_app.config)
db.put_doc(key, v2_node.values()[0]['layerType'], v2_request, layer_config=v2_node, v1_config=v1_node)
except Exception as e:
msg = {'msg': 'Error: {0}'.format(e.message)}
current_app.logger.error('Failed to upgrade {0}'.format(key), exc_info=e)
return Response(json.dumps(msg), mimetype='application/json', status=400)
success = {"msg": "Upgraded", "version": "2.0", "key": key}
return Response(json.dumps(success), mimetype='application/json', status=200)
def put(self, smallkey):
"""
A REST endpoint for adding or editing a single layer.
All registration requests must contain entries for all languages and will be validated against a JSON schema.
:param smallkey: A unique identifier for the dataset (can be any unique string, but preferably should be short)
:type smallkey: str
:returns: JSON Response -- 201 on success; 400 with JSON payload of an errors array on failure
"""
try:
s = json.loads( request.data )
except Exception:
return '{"errors":["Unparsable json"]}',400
if not validator.is_valid( s ):
resp = { 'errors': [x.message for x in validator.iter_errors(s)] }
app.logger.info( resp )
return Response(json.dumps(resp), mimetype='application/json', status=400)
data = dict( key=smallkey, request=s )
try:
if s['payload_type'] == 'wms':
data['en'] = regparse.wms.make_node( s['en'], regparse.make_id(smallkey,'en'), app.config )
data['fr'] = regparse.wms.make_node( s['fr'], regparse.make_id(smallkey,'fr'), app.config )
else:
data['en'] = regparse.esri_feature.make_node( s['en'], regparse.make_id(smallkey,'en'), app.config )
data['fr'] = regparse.esri_feature.make_node( s['fr'], regparse.make_id(smallkey,'fr'), app.config )
except regparse.metadata.MetadataException as mde:
app.logger.warning( 'Metadata could not be retrieved for layer', exc_info=mde )
abort( 400, msg=mde.message )
app.logger.debug( data )
db.put_doc( smallkey, { 'type':s['payload_type'], 'data':data } )
app.logger.info( 'added a smallkey %s' % smallkey )
return smallkey, 201
def refresh_records( day_limit, config ):
import db, datetime, string
valid = []
invalid = {}
query = ""
if day_limit is None:
query = "function(doc) { emit(doc._id, { updated: doc.updated_at, key: doc.data.key, request: doc.data.request }); }"
else:
min_age = datetime.date.today() - datetime.timedelta( days=day_limit )
query = "function(doc) { if (doc.updated_at <= '$date') emit(doc._id, { updated: doc.updated_at, key: doc.data.key, request: doc.data.request }); }"
query = string.Template(query).substitute( date=min_age )
results = db.query( query )
for r in results:
key = r['id']
print r
if 'request' not in r['value']:
invalid[key] = 'previous request was not cached (request caching added in 1.8.0)'
continue
req = r['value']['request']
try:
data = dict( key=key, request=req )
if req['payload_type'] == 'wms':
data['en'] = wms.make_node( req['en'], make_id(key,'en'), config )
data['fr'] = wms.make_node( req['fr'], make_id(key,'fr'), config )
else:
data['en'] = esri_feature.make_node( req['en'], make_id(key,'en'), config )
data['fr'] = esri_feature.make_node( req['fr'], make_id(key,'fr'), config )
db.put_doc( key, { 'type':req['payload_type'], 'data':data } )
valid.append( key )
except Exception as e:
invalid[key] = str(e)
return { "updated":valid, "errors":invalid }
def put(self, key):
"""
A REST endpoint for updating one or more nodes of an existing request.
:param key: A unique identifier for the dataset
:type key: str
:returns: JSON Response -- 200 on success; 4xx if problems are encountered
"""
langs = ["en", "fr"]
try:
dbdata = db.get_raw(key)
if dbdata is None:
return '{"msg":"Record not found in database"}', 404
except Exception as e:
msg = {'msg': 'Error: {0}'.format(e.message)}
return Response(json.dumps(msg),
mimetype='application/json',
status=500)
def replace_if_set(params):
for p in params:
if p in payload_seg:
request_seg[p] = payload_seg[p]
try:
payload = json.loads(request.data)
for lang in langs:
request_seg = dbdata['request'][lang]
payload_seg = payload[lang]
replace_if_set(['service_url', 'service_name', 'metadata'])
if payload_seg['service_type'] in [
'esriFeature', 'esriImage', 'esriTile'
]:
replace_if_set(['display_field'])
elif payload_seg['service_type'] == 'esriMapServer':
replace_if_set(['scrape_only', 'recursive'])
elif payload_seg['service_type'] == 'ogcWms':
replace_if_set([
'scrape_only', 'recursive', 'legend_format',
'feature_info_format'
])
v2_node, v1_node = regparse.make_node(key, dbdata["request"],
current_app.config)
db.put_doc(key,
payload[lang]["service_type"],
dbdata["request"],
layer_config=v2_node,
v1_config=v1_node)
except Exception as e:
msg = {'msg': 'Error: {0}'.format(e.message)}
return Response(json.dumps(msg),
mimetype='application/json',
status=500)
success = {"msg": "Updated", "key": key}
return Response(json.dumps(success),
mimetype='application/json',
status=200)
def cl_submit_metadata(dir):
all_docs = get_all_docs(dir)
print("""Write medata line by line as key value pairs.
Write lists comma separated.
A blank line indicates the end of the metadata.\n""")
for file_path in all_docs:
metadata = collect_metadata(file_path)
db.put_doc(file_path, metadata)
def refresh_records(day_limit, limit, config):
import datetime, string
valid = []
invalid = {}
query = ""
if day_limit is None:
query = "function(doc) { emit(doc._id, { updated: doc.updated_at, version: doc.version, request: doc.data ? doc.data.request : doc.request }); }" # NOQA
else:
min_age = datetime.date.today() - datetime.timedelta(days=day_limit)
query = "function(doc) { if (doc.updated_at <= '$date') emit(doc._id, { updated: doc.updated_at, version: doc.version, request: doc.data ? doc.data.request : doc.request }); }" # NOQA
query = string.Template(query).substitute(date=min_age)
current_app.logger.debug('CouchDB Query {0}'.format(query))
results = db.query(query)
count = 0
limit_reached = False
for r in results:
if limit is not None:
# annoying count is necessary since we don't want to convert a generator into a list
count += 1
if count > limit:
limit_reached = True
break
key = r['id']
current_app.logger.info('refresh {0}'.format(key))
if 'request' not in r['value']:
invalid[
key] = 'previous request was not cached (request caching added in 1.8.0)'
continue
if r['value'].get('version', None) != '2.0':
invalid[
key] = 'v1 record found; upgrading record types has not yet been implemented'
continue
req = r['value']['request']
try:
v2_node, v1_node = regparse.make_node(key, req, config)
db.put_doc(key,
v2_node.values()[0]['layerType'],
req,
layer_config=v2_node,
v1_config=v1_node)
valid.append(key)
except Exception as e:
current_app.logger.warning('Error in refresh', exc_info=e)
invalid[key] = str(e)
return {
"updated": valid,
"errors": invalid,
'limit_reached': limit_reached
}
def post(self, key):
"""
A REST endpoint for upgrading a previous registration from v1 to v2.
:param key: A unique identifier for the dataset
:type key: str
:returns: JSON Response -- 200 on success; 4xx if problems are encountered
"""
dbdata = db.get_raw(key)
if dbdata is None:
return '{"msg":"Record not found in database"}', 404
elif dbdata.get('version') == '2.0':
return '{"msg":"Already upgraded"}', 200
elif dbdata['data'].get('request') is None:
return '{"msg":"Previous request was not cached (request caching added in 1.8.0)"}', 409
try:
v1_request = dbdata['data']['request']
upgrade_method = wms_upgrade if v1_request[
'payload_type'] == 'wms' else feat_upgrade
v2_request = {
lang: upgrade_method(v1_request[lang])
for lang in current_app.config['LANGS']
}
print v2_request
v2_node, v1_node = regparse.make_node(key, v2_request,
current_app.config)
db.put_doc(key,
v2_node.values()[0]['layerType'],
v2_request,
layer_config=v2_node,
v1_config=v1_node)
except Exception as e:
msg = {'msg': 'Error: {0}'.format(e.message)}
current_app.logger.error('Failed to upgrade {0}'.format(key),
exc_info=e)
return Response(json.dumps(msg),
mimetype='application/json',
status=400)
success = {"msg": "Upgraded", "version": "2.0", "key": key}
return Response(json.dumps(success),
mimetype='application/json',
status=200)
def post(self, smallkey):
"""
A REST endpoint for updating details in a feature layer.
:param smallkey: A unique identifier for the dataset (can be any unique string, but preferably should be short)
:type smallkey: str
:returns: JSON Response -- 200 on success; 400 with JSON payload of an errors array on failure
"""
try:
payload = json.loads(request.data)
except Exception:
return '{"errors":["Unparsable json"]}', 400
fragment = {'en': {}, 'fr': {}}
if len(payload) == 2 and 'en' in payload and 'fr' in payload:
fragment = payload
else:
fragment['en'].update(payload)
fragment['fr'].update(payload)
dbdata = db.get_raw(smallkey)
if dbdata is None:
return '{"errors":["Record not found in database"]}', 404
elif dbdata['type'] != 'feature':
return '{"errors":["Record is not a feature layer"]}', 400
dbdata['data']['request']['en'].update(fragment['en'])
dbdata['data']['request']['fr'].update(fragment['fr'])
errors = get_registration_errors(payload)
if errors:
resp = {'errors': errors}
current_app.logger.info(resp)
return Response(json.dumps(resp), mimetype='application/json', status=400)
try:
data = regparse.make_record(smallkey, dbdata['data']['request'], current_app.config)
except regparse.metadata.MetadataException as mde:
current_app.logger.warning('Metadata could not be retrieved for layer', exc_info=mde)
abort(400, msg=mde.message)
db.put_doc(smallkey, {'type': data['request']['payload_type'], 'data': data})
return smallkey, 200
def put(self, smallkey):
"""
A REST endpoint for updating a simplification factor on a registered feature service.
:param smallkey: A unique identifier for the dataset (can be any unique string, but preferably should be short)
:type smallkey: str
:returns: JSON Response -- 200 on success; 400 with JSON payload of an errors array on failure
"""
try:
payload = json.loads( request.data )
except Exception:
return '{"errors":["Unparsable json"]}',400
#check that our payload has a 'factor' property that contains an integer
if not isinstance(payload['factor'], numbers.Integral):
resp = { 'errors': ['Invalid payload JSON'] }
app.logger.info( resp )
return Response(json.dumps(resp), mimetype='application/json', status=400)
intFactor = int( payload['factor'] )
#grab english and french doc fragments
dbdata = db.get_raw( smallkey )
if dbdata is None:
#smallkey/lang is not in the database
return '{"errors":["Record not found in database"]}',404
elif dbdata['type'] != 'feature':
#layer is not a feature layer
return '{"errors":["Record is not a feature layer"]}',400
else:
#add in new simplification factor
dbdata['data']['en']['maxAllowableOffset'] = intFactor
dbdata['data']['fr']['maxAllowableOffset'] = intFactor
#also store factor in the request, so we can preserve the factor during an update
dbdata['data']['request']['en']['maxAllowableOffset'] = intFactor
dbdata['data']['request']['fr']['maxAllowableOffset'] = intFactor
#put back in the database
db.put_doc( smallkey, { 'type':dbdata['type'], 'data':dbdata['data'] } )
app.logger.info( 'updated simpification factor on smallkey %(s)s to %(f)d by %(u)s' % {"s":smallkey, "f": intFactor, "u": payload['user'] } )
return smallkey, 200
def put(self, smallkey):
"""
A REST endpoint for updating a simplification factor on a registered feature service.
:param smallkey: A unique identifier for the dataset (can be any unique string, but preferably should be short)
:type smallkey: str
:returns: JSON Response -- 200 on success; 400 with JSON payload of an errors array on failure
"""
try:
payload = json.loads(request.data)
except Exception:
return '{"errors":["Unparsable json"]}', 400
# Check that our payload has a 'factor' property that contains an integer
if not isinstance(payload['factor'], numbers.Integral):
resp = {'errors': ['Invalid payload JSON']}
current_app.logger.info(resp)
return Response(json.dumps(resp), mimetype='application/json', status=400)
intFactor = int(payload['factor'])
# Grab english and french doc fragments
dbdata = db.get_raw(smallkey)
if dbdata is None:
return '{"errors":["Record not found in database"]}', 404
elif dbdata['type'] != 'feature':
return '{"errors":["Record is not a feature layer"]}', 400
else:
# Add in new simplification factor
dbdata['data']['en']['maxAllowableOffset'] = intFactor
dbdata['data']['fr']['maxAllowableOffset'] = intFactor
# Also store factor in the request, so we can preserve the factor during an update
dbdata['data']['request']['en']['max_allowable_offset'] = intFactor
dbdata['data']['request']['fr']['max_allowable_offset'] = intFactor
# Update the database record
db.put_doc(smallkey, {'type': dbdata['type'], 'data': dbdata['data']})
current_app.logger.info('updated simpification factor on smallkey %(s)s to %(f)d by %(u)s'
% {"s": smallkey, "f": intFactor, "u": payload['user']})
return smallkey, 200
def put(self, key):
"""
A REST endpoint for adding or editing a single layer.
All registration requests must contain entries for all languages and will be validated against a JSON schema.
:param smallkey: A unique identifier for the dataset (can be any unique string, but preferably should be short)
:type smallkey: str
:returns: JSON Response -- 201 on success; 400 with JSON payload of an errors array on failure
"""
try:
req = json.loads(request.data)
except Exception as e:
current_app.logger.error(e.message)
return '{"errors":["Unparsable json"]}', 400
errors = get_registration_errors(req)
if errors:
resp = {'errors': errors}
current_app.logger.info(resp)
return Response(json.dumps(resp),
mimetype='application/json',
status=400)
try:
v2_node, v1_node = regparse.make_node(key, req, current_app.config)
except regparse.metadata.MetadataException as mde:
current_app.logger.warning(
'Metadata could not be retrieved for layer', exc_info=mde)
abort(400, msg=mde.message)
except regparse.ServiceEndpointException as se:
current_app.logger.warning('Problem reading service endpoints',
exc_info=se)
abort(400, msg=se.message)
current_app.logger.debug(v2_node)
current_app.logger.debug(v1_node)
db.put_doc(key,
v2_node.values()[0]['layerType'],
req,
layer_config=v2_node,
v1_config=v1_node)
current_app.logger.info('added a key %s' % key)
return Response(json.dumps(dict(key=key)),
mimetype='application/json',
status=201)
def put(self, key):
"""
A REST endpoint for updating one or more nodes of an existing request.
:param key: A unique identifier for the dataset
:type key: str
:returns: JSON Response -- 200 on success; 4xx if problems are encountered
"""
langs = ["en", "fr"]
try:
dbdata = db.get_raw(key)
if dbdata is None:
return '{"msg":"Record not found in database"}', 404
except Exception as e:
msg = {'msg': 'Error: {0}'.format(e.message)}
return Response(json.dumps(msg), mimetype='application/json', status=500)
def replace_if_set(params):
for p in params:
if p in payload_seg:
request_seg[p] = payload_seg[p]
try:
payload = json.loads(request.data)
for lang in langs:
request_seg = dbdata['request'][lang]
payload_seg = payload[lang]
replace_if_set(['service_url', 'service_name', 'metadata'])
if payload_seg['service_type'] in ['esriFeature', 'esriImage', 'esriTile']:
replace_if_set(['display_field'])
elif payload_seg['service_type'] == 'esriMapServer':
replace_if_set(['scrape_only', 'recursive'])
elif payload_seg['service_type'] == 'ogcWms':
replace_if_set(['scrape_only', 'recursive', 'legend_format', 'feature_info_format'])
v2_node, v1_node = regparse.make_node(key, dbdata["request"], current_app.config)
db.put_doc(key, payload[lang]["service_type"], dbdata["request"],
layer_config=v2_node, v1_config=v1_node)
except Exception as e:
msg = {'msg': 'Error: {0}'.format(e.message)}
return Response(json.dumps(msg), mimetype='application/json', status=500)
success = {"msg": "Updated", "key": key}
return Response(json.dumps(success), mimetype='application/json', status=200)
def refresh_records(day_limit, limit, config):
import datetime, string
valid = []
invalid = {}
query = ""
if day_limit is None:
query = "function(doc) { emit(doc._id, { updated: doc.updated_at, version: doc.version, request: doc.data ? doc.data.request : doc.request }); }" # NOQA
else:
min_age = datetime.date.today() - datetime.timedelta(days=day_limit)
query = "function(doc) { if (doc.updated_at <= '$date') emit(doc._id, { updated: doc.updated_at, version: doc.version, request: doc.data ? doc.data.request : doc.request }); }" # NOQA
query = string.Template(query).substitute(date=min_age)
current_app.logger.debug('CouchDB Query {0}'.format(query))
results = db.query(query)
count = 0
limit_reached = False
for r in results:
if limit is not None:
# annoying count is necessary since we don't want to convert a generator into a list
count += 1
if count > limit:
limit_reached = True
break
key = r['id']
current_app.logger.info('refresh {0}'.format(key))
if 'request' not in r['value']:
invalid[key] = 'previous request was not cached (request caching added in 1.8.0)'
continue
if r['value'].get('version', None) != '2.0':
invalid[key] = 'v1 record found; upgrading record types has not yet been implemented'
continue
req = r['value']['request']
try:
v2_node, v1_node = regparse.make_node(key, req, config)
db.put_doc(key, v2_node.values()[0]['layerType'], req, layer_config=v2_node, v1_config=v1_node)
valid.append(key)
except Exception as e:
current_app.logger.warning('Error in refresh', exc_info=e)
invalid[key] = str(e)
return {"updated": valid, "errors": invalid, 'limit_reached': limit_reached}