def run_import(upload_session, async):
"""Run the import, possibly asynchronously.
Returns the target datastore.
"""
import_session = upload_session.import_session
import_session = gs_uploader.get_session(import_session.id)
task = import_session.tasks[0]
if import_session.state == 'INCOMPLETE':
if task.state != 'ERROR':
raise Exception('unknown item state: %s' % task.state)
elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore':
if task.state == 'READY':
import_session.commit(async)
elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore':
if task.state == 'READY':
import_session.commit(async)
# if a target datastore is configured, ensure the datastore exists
# in geoserver and set the uploader target appropriately
if ogc_server_settings.GEOGIG_ENABLED and upload_session.geogig is True \
and task.target.store_type != 'coverageStore':
target = create_geoserver_db_featurestore(
store_type='geogig',
store_name=upload_session.geogig_store,
author_name=upload_session.user.username,
author_email=upload_session.user.email)
_log(
'setting target datastore %s %s',
target.name,
target.workspace.name)
task.set_target(target.name, target.workspace.name)
elif ogc_server_settings.datastore_db and task.target.store_type != 'coverageStore':
target = create_geoserver_db_featurestore()
_log(
'setting target datastore %s %s',
target.name,
target.workspace.name)
task.set_target(target.name, target.workspace.name)
else:
target = task.target
if upload_session.update_mode:
_log('setting updateMode to %s', upload_session.update_mode)
task.set_update_mode(upload_session.update_mode)
_log('running import session')
# run async if using a database
import_session.commit(async)
# @todo check status of import session - it may fail, but due to protocol,
# this will not be reported during the commit
return target
def run_import(upload_session, async):
"""Run the import, possibly asynchronously.
Returns the target datastore.
"""
import_session = upload_session.import_session
import_session = gs_uploader.get_session(import_session.id)
task = import_session.tasks[0]
import_execution_requested = False
if import_session.state == 'INCOMPLETE':
if task.state != 'ERROR':
raise Exception('unknown item state: %s' % task.state)
elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore':
if task.state == 'READY':
import_session.commit(async)
import_execution_requested = True
# if a target datastore is configured, ensure the datastore exists
# in geoserver and set the uploader target appropriately
if ogc_server_settings.GEOGIG_ENABLED and upload_session.geogig is True \
and task.target.store_type != 'coverageStore':
target = create_geoserver_db_featurestore(
store_type='geogig',
store_name=upload_session.geogig_store,
author_name=upload_session.user.username,
author_email=upload_session.user.email)
_log(
'setting target datastore %s %s',
target.name,
target.workspace.name)
task.set_target(target.name, target.workspace.name)
elif ogc_server_settings.datastore_db and task.target.store_type != 'coverageStore':
target = create_geoserver_db_featurestore()
_log(
'setting target datastore %s %s',
target.name,
target.workspace.name)
task.set_target(target.name, target.workspace.name)
else:
target = task.target
if upload_session.update_mode:
_log('setting updateMode to %s', upload_session.update_mode)
task.set_update_mode(upload_session.update_mode)
_log('running import session')
# run async if using a database
if not import_execution_requested:
import_session.commit(async)
# @todo check status of import session - it may fail, but due to protocol,
# this will not be reported during the commit
return target
def run_import(upload_session, async):
"""Run the import, possibly asynchronously.
Returns the target datastore.
"""
import_session = upload_session.import_session
import_session = gs_uploader().get_session(import_session.id)
if import_session.state == 'INCOMPLETE':
item = upload_session.import_session.tasks[0].items[0]
if item.state == 'NO_CRS':
err = 'No projection found'
else:
err = item.state or 'Session not ready for import.'
if err:
raise Exception(err)
# if a target datastore is configured, ensure the datastore exists
# in geoserver and set the uploader target appropriately
if (hasattr(settings, 'GEOGIT_DATASTORE') and settings.GEOGIT_DATASTORE and
upload_session.geogit == True and
import_session.tasks[0].items[0].layer.layer_type != 'RASTER'):
target = create_geoserver_db_featurestore(store_type='geogit')
_log('setting target datastore %s %s',
target.name, target.workspace.name
)
import_session.tasks[0].set_target(
target.name, target.workspace.name)
elif (settings.DB_DATASTORE and
import_session.tasks[0].items[0].layer.layer_type != 'RASTER'):
target = create_geoserver_db_featurestore(store_type='postgis')
_log('setting target datastore %s %s',
target.name, target.workspace.name
)
import_session.tasks[0].set_target(
target.name, target.workspace.name)
else:
target = import_session.tasks[0].target
if upload_session.update_mode:
_log('setting updateMode to %s', upload_session.update_mode)
import_session.tasks[0].set_update_mode(upload_session.update_mode)
_log('running import session')
# run async if using a database
import_session.commit(async)
# @todo check status of import session - it may fail, but due to protocol,
# this will not be reported during the commit
return target
def run_import(upload_session, async):
"""Run the import, possibly asynchronously.
Returns the target datastore.
"""
import_session = upload_session.import_session
import_session = gs_uploader().get_session(import_session.id)
if import_session.state == 'INCOMPLETE':
item = upload_session.import_session.tasks[0].items[0]
if item.state == 'NO_CRS':
err = 'No projection found'
else:
err = item.state or 'Session not ready for import.'
if err:
raise Exception(err)
# if a target datastore is configured, ensure the datastore exists
# in geoserver and set the uploader target appropriately
if (hasattr(settings, 'GEOGIT_DATASTORE') and settings.GEOGIT_DATASTORE and
upload_session.geogit == True and
import_session.tasks[0].items[0].layer.layer_type != 'RASTER'):
target = create_geoserver_db_featurestore(store_type='geogit', store_name = upload_session.geogit_store)
_log('setting target datastore %s %s',
target.name, target.workspace.name
)
import_session.tasks[0].set_target(
target.name, target.workspace.name)
elif (settings.OGC_SERVER['default']['OPTIONS']['DATASTORE'] != '' and
import_session.tasks[0].items[0].layer.layer_type != 'RASTER'):
target = create_geoserver_db_featurestore(store_type='postgis', store_name = upload_session.geogit_store)
_log('setting target datastore %s %s',
target.name, target.workspace.name
)
import_session.tasks[0].set_target(
target.name, target.workspace.name)
else:
target = import_session.tasks[0].target
if upload_session.update_mode:
_log('setting updateMode to %s', upload_session.update_mode)
import_session.tasks[0].set_update_mode(upload_session.update_mode)
_log('running import session')
# run async if using a database
import_session.commit(async)
# @todo check status of import session - it may fail, but due to protocol,
# this will not be reported during the commit
return target
def layer_create(request, data=None, template='upload/layer_create.html'):
print 'layer create'
if request.method == 'POST':
feature_type = json.loads(request.POST.get(u'featureType', None))
datastore = feature_type['store']['name']
store_create_geogig = json.loads(request.POST.get(u'storeCreateGeogig', 'false'))
if store_create_geogig:
if not feature_type['store']['name']:
store_proposed = slugify(' '.join([request.user.username, feature_type['name']]))
store_created = create_geoserver_db_featurestore(store_type='geogig', store_name=store_proposed)
feature_type['store']['name'] = store_created.name
# -- only allow creation of layers in the whitelisted datastores
try:
settings.ALLOWED_DATASTORE_LAYER_CREATE
except AttributeError:
return HttpResponse(status=FORBIDDEN, content='ALLOWED_DATASTORE_LAYER_CREATE whitelist is not defined.')
if datastore not in settings.ALLOWED_DATASTORE_LAYER_CREATE:
return HttpResponse(status=FORBIDDEN, content='datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.')
post_request = requests.post(
'{}/workspaces/{}/datastores/{}/featuretypes.json'.format(ogc_server_settings.rest, feature_type['namespace']['name'], datastore),
data='{{"featureType":{}}}'.format(json.dumps(feature_type)),
auth=ogc_server_settings.credentials,
headers={'content-type': 'application/json'}
)
if post_request.status_code == 200 or post_request.status_code == 201:
# import the layer from geoserver to geonode
response = gs_slurp(filter=feature_type['name'], workspace=feature_type['namespace']['name'], store=feature_type['store']['name'])
if 'layers' in response and len(response['layers']) == 1 and 'name' in response['layers'][0] and response['layers'][0]['name'] == feature_type['name']:
# configruer layer with time dimension as a list for now. Since importer is being refactored,
# we'll change how it is done.
layer = gs_catalog.get_layer(feature_type['name'])
resource = layer.resource
if layer:
time_info = DimensionInfo(
name='time',
enabled=True,
presentation='LIST',
resolution=None,
units=None,
unitSymbol=None,
attribute='time',
end_attribute=None,
strategy=None)
resource.metadata = {'time': time_info}
resource.catalog.save(resource)
return HttpResponse(status=post_request.status_code, content=post_request.text)
return HttpResponse(status=500, content='failed to configure layer')
else:
return HttpResponse(status=500, content='failed to add created layer from geoserver to geonode')
else:
return HttpResponse(status=post_request.status_code, content=post_request.text)
print '---- create layer response: ', post_request.text
return render_to_response(template, RequestContext(request, {}))
def import_file(self, *args, **kwargs):
"""
Creates a Geoserver Layer from JSON.
{
"name": "a new layer",
"store": {"name": "store"},
"namespace": {"name": 'name'},
"attributes": {"attribute": [
{"name": "time",
"binding": "org.geotools.data.postgis.PostGISDialect$XDate",
"minOccurs": 0,
"nillable": True},
{"name": "geometry",
"binding": "com.vividsolutions.jts.geom.MultiLineString",
"minOccurs":0,
"nillable":True}]},
"nativeCRS": "EPSG:4326",
"srs": "EPSG:4326"
}
"""
self.completed_layers = []
configuration_options = kwargs.get('configuration_options')
if not configuration_options:
return
# Configuration options should be a list at this point since the importer can process multiple layers in a
# single import
if isinstance(configuration_options, dict):
configuration_options = [configuration_options]
for layer in configuration_options:
feature_type = layer.get('featureType')
store_name = feature_type['store']['name']
store_create_geogig = layer.get(u'storeCreateGeogig', 'false')
owner = layer.get('layer_owner')
username = owner.username
email = owner.email
layer['layer_type'] = 'vector'
layer['fields'] = None
layer['geoserver_store'] = {'type': 'geogig'}
store = layer.get('geoserver_store')
if store.get('type', str).lower() == 'geogig':
name = slugify(feature_type['name'])
store.setdefault('branch', 'master')
store.setdefault('create', 'true')
store.setdefault('name', name)
store['geogig_repository'] = ("geoserver://%s" % name)
store_name = name
feature_type['title'] = feature_type['name']
feature_type['name'] = launder(slugify(unicode(feature_type['name'])))
# Without this check, the bounding box will default to 0, 0, -1, -1
if 'nativeBoundingBox' not in feature_type:
feature_type['nativeBoundingBox'] = {'minx': -180, 'maxx': 180, 'miny': -90, 'maxy': 90, 'crs': 'EPSG:4326'}
if store_create_geogig and store_create_geogig != 'false':
store_created = create_geoserver_db_featurestore(store_type='geogig', store_name=store_name, author_name=username,
author_email=email)
feature_type['store']['name'] = store_created.name
if store_name not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []) and '*' not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []):
return HttpResponseForbidden(content='Datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.')
for x in range(0, 50):
post_request = requests.post(
'{}/workspaces/{}/datastores/{}/featuretypes.json'.format(ogc_server_settings.rest, feature_type['namespace']['name'], store_name),
data='{{"featureType":{}}}'.format(json.dumps(feature_type)),
auth=ogc_server_settings.credentials,
headers={'content-type': 'application/json'}
)
if post_request.ok:
break
if 'already exists in' in post_request.content:
break
time.sleep(3)
if post_request.ok:
self.completed_layers.append([feature_type['name'], layer])
else:
message = 'Unable to create layer: {0}, an unhandled exception occurred.'.format(feature_type['name'])
if 'already exists in' in post_request.content:
message = 'A layer named {0} already exists. Please choose another name.'.format(feature_type['name'])
print post_request.content
raise UploadError(message)
return self.completed_layers
def create_datastore(store_name=None, store_type=None):
if not store_name:
store_name = ogc_server_settings.datastore_db['NAME']
return create_geoserver_db_featurestore(
store_type=store_type, store_name=store_name)
def import_file(self, *args, **kwargs):
"""
Creates a Geoserver Layer from JSON.
{
"name": "a new layer",
"store": {"name": "store"},
"namespace": {"name": 'name'},
"attributes": {"attribute": [
{"name": "time",
"binding": "org.geotools.data.postgis.PostGISDialect$XDate",
"minOccurs": 0,
"nillable": True},
{"name": "geometry",
"binding": "com.vividsolutions.jts.geom.MultiLineString",
"minOccurs":0,
"nillable":True}]},
"nativeCRS": "EPSG:4326",
"srs": "EPSG:4326"
}
"""
self.completed_layers = []
configuration_options = kwargs.get('configuration_options')
if not configuration_options:
return
# Configuration options should be a list at this point since the importer can process multiple layers in a
# single import
if isinstance(configuration_options, dict):
configuration_options = [configuration_options]
for layer in configuration_options:
feature_type = layer.get('featureType')
datastore = feature_type['store']['name']
store_create_geogig = layer.get(u'storeCreateGeogig', 'false')
feature_type['title'] = feature_type['name']
feature_type['name'] = launder(slugify(unicode(feature_type['name'])))
# Without this check, the bounding box will default to 0, 0, -1, -1
if 'nativeBoundingBox' not in feature_type:
feature_type['nativeBoundingBox'] = {'minx': -180, 'maxx': 180, 'miny': -90, 'maxy': 90, 'crs': 'EPSG:4326'}
if store_create_geogig and store_create_geogig != 'false':
store_created = create_geoserver_db_featurestore(store_type='geogig', store_name=feature_type['store']['name'])
feature_type['store']['name'] = store_created.name
if datastore not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []) and '*' not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []):
return HttpResponseForbidden(content='Datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.')
post_request = requests.post(
'{}/workspaces/{}/datastores/{}/featuretypes.json'.format(ogc_server_settings.rest, feature_type['namespace']['name'], datastore),
data='{{"featureType":{}}}'.format(json.dumps(feature_type)),
auth=ogc_server_settings.credentials,
headers={'content-type': 'application/json'}
)
if post_request.ok:
self.completed_layers.append([feature_type['name'], layer])
else:
message = 'Unable to create layer: {0}, an unhandled exception occurred.'.format(feature_type['name'])
if 'already exists in' in post_request.content:
message = 'A layer named {0} already exists. Please choose another name.'.format(feature_type['name'])
print post_request.content
raise UploadError(message)
return self.completed_layers
def layer_create(request, data=None, template='upload/layer_create.html'):
print 'layer create'
if request.method == 'POST':
feature_type = json.loads(request.POST.get(u'featureType', None))
datastore = feature_type['store']['name']
store_create_geogig = json.loads(
request.POST.get(u'storeCreateGeogig', 'false'))
if store_create_geogig:
if not feature_type['store']['name']:
store_proposed = slugify(' '.join(
[request.user.username, feature_type['name']]))
store_created = create_geoserver_db_featurestore(
store_type='geogig', store_name=store_proposed)
feature_type['store']['name'] = store_created.name
# -- only allow creation of layers in the whitelisted datastores
try:
settings.ALLOWED_DATASTORE_LAYER_CREATE
except AttributeError:
return HttpResponse(
status=FORBIDDEN,
content=
'ALLOWED_DATASTORE_LAYER_CREATE whitelist is not defined.')
if datastore not in settings.ALLOWED_DATASTORE_LAYER_CREATE:
return HttpResponse(
status=FORBIDDEN,
content=
'datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.'
)
post_request = requests.post(
'{}/workspaces/{}/datastores/{}/featuretypes.json'.format(
ogc_server_settings.rest, feature_type['namespace']['name'],
datastore),
data='{{"featureType":{}}}'.format(json.dumps(feature_type)),
auth=ogc_server_settings.credentials,
headers={'content-type': 'application/json'})
if post_request.status_code == 200 or post_request.status_code == 201:
# import the layer from geoserver to geonode
response = gs_slurp(filter=feature_type['name'],
workspace=feature_type['namespace']['name'],
store=feature_type['store']['name'])
if 'layers' in response and len(
response['layers']
) == 1 and 'name' in response['layers'][0] and response['layers'][
0]['name'] == feature_type['name']:
# configruer layer with time dimension as a list for now. Since importer is being refactored,
# we'll change how it is done.
layer = gs_catalog.get_layer(feature_type['name'])
resource = layer.resource
if layer:
time_info = DimensionInfo(name='time',
enabled=True,
presentation='LIST',
resolution=None,
units=None,
unitSymbol=None,
attribute='time',
end_attribute=None,
strategy=None)
resource.metadata = {'time': time_info}
resource.catalog.save(resource)
return HttpResponse(status=post_request.status_code,
content=post_request.text)
return HttpResponse(status=500,
content='failed to configure layer')
else:
return HttpResponse(
status=500,
content=
'failed to add created layer from geoserver to geonode')
else:
return HttpResponse(status=post_request.status_code,
content=post_request.text)
print '---- create layer response: ', post_request.text
return render_to_response(template, RequestContext(request, {}))