def run_import(upload_session, async): """Run the import, possibly asynchronously. Returns the target datastore. """ import_session = upload_session.import_session import_session = gs_uploader.get_session(import_session.id) task = import_session.tasks[0] if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore': if task.state == 'READY': import_session.commit(async) elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore': if task.state == 'READY': import_session.commit(async) # if a target datastore is configured, ensure the datastore exists # in geoserver and set the uploader target appropriately if ogc_server_settings.GEOGIG_ENABLED and upload_session.geogig is True \ and task.target.store_type != 'coverageStore': target = create_geoserver_db_featurestore( store_type='geogig', store_name=upload_session.geogig_store, author_name=upload_session.user.username, author_email=upload_session.user.email) _log( 'setting target datastore %s %s', target.name, target.workspace.name) task.set_target(target.name, target.workspace.name) elif ogc_server_settings.datastore_db and task.target.store_type != 'coverageStore': target = create_geoserver_db_featurestore() _log( 'setting target datastore %s %s', target.name, target.workspace.name) task.set_target(target.name, target.workspace.name) else: target = task.target if upload_session.update_mode: _log('setting updateMode to %s', upload_session.update_mode) task.set_update_mode(upload_session.update_mode) _log('running import session') # run async if using a database import_session.commit(async) # @todo check status of import session - it may fail, but due to protocol, # this will not be reported during the commit return target
def run_import(upload_session, async): """Run the import, possibly asynchronously. Returns the target datastore. """ import_session = upload_session.import_session import_session = gs_uploader.get_session(import_session.id) task = import_session.tasks[0] import_execution_requested = False if import_session.state == 'INCOMPLETE': if task.state != 'ERROR': raise Exception('unknown item state: %s' % task.state) elif import_session.state == 'PENDING' and task.target.store_type == 'coverageStore': if task.state == 'READY': import_session.commit(async) import_execution_requested = True # if a target datastore is configured, ensure the datastore exists # in geoserver and set the uploader target appropriately if ogc_server_settings.GEOGIG_ENABLED and upload_session.geogig is True \ and task.target.store_type != 'coverageStore': target = create_geoserver_db_featurestore( store_type='geogig', store_name=upload_session.geogig_store, author_name=upload_session.user.username, author_email=upload_session.user.email) _log( 'setting target datastore %s %s', target.name, target.workspace.name) task.set_target(target.name, target.workspace.name) elif ogc_server_settings.datastore_db and task.target.store_type != 'coverageStore': target = create_geoserver_db_featurestore() _log( 'setting target datastore %s %s', target.name, target.workspace.name) task.set_target(target.name, target.workspace.name) else: target = task.target if upload_session.update_mode: _log('setting updateMode to %s', upload_session.update_mode) task.set_update_mode(upload_session.update_mode) _log('running import session') # run async if using a database if not import_execution_requested: import_session.commit(async) # @todo check status of import session - it may fail, but due to protocol, # this will not be reported during the commit return target
def run_import(upload_session, async): """Run the import, possibly asynchronously. Returns the target datastore. """ import_session = upload_session.import_session import_session = gs_uploader().get_session(import_session.id) if import_session.state == 'INCOMPLETE': item = upload_session.import_session.tasks[0].items[0] if item.state == 'NO_CRS': err = 'No projection found' else: err = item.state or 'Session not ready for import.' if err: raise Exception(err) # if a target datastore is configured, ensure the datastore exists # in geoserver and set the uploader target appropriately if (hasattr(settings, 'GEOGIT_DATASTORE') and settings.GEOGIT_DATASTORE and upload_session.geogit == True and import_session.tasks[0].items[0].layer.layer_type != 'RASTER'): target = create_geoserver_db_featurestore(store_type='geogit') _log('setting target datastore %s %s', target.name, target.workspace.name ) import_session.tasks[0].set_target( target.name, target.workspace.name) elif (settings.DB_DATASTORE and import_session.tasks[0].items[0].layer.layer_type != 'RASTER'): target = create_geoserver_db_featurestore(store_type='postgis') _log('setting target datastore %s %s', target.name, target.workspace.name ) import_session.tasks[0].set_target( target.name, target.workspace.name) else: target = import_session.tasks[0].target if upload_session.update_mode: _log('setting updateMode to %s', upload_session.update_mode) import_session.tasks[0].set_update_mode(upload_session.update_mode) _log('running import session') # run async if using a database import_session.commit(async) # @todo check status of import session - it may fail, but due to protocol, # this will not be reported during the commit return target
def run_import(upload_session, async): """Run the import, possibly asynchronously. Returns the target datastore. """ import_session = upload_session.import_session import_session = gs_uploader().get_session(import_session.id) if import_session.state == 'INCOMPLETE': item = upload_session.import_session.tasks[0].items[0] if item.state == 'NO_CRS': err = 'No projection found' else: err = item.state or 'Session not ready for import.' if err: raise Exception(err) # if a target datastore is configured, ensure the datastore exists # in geoserver and set the uploader target appropriately if (hasattr(settings, 'GEOGIT_DATASTORE') and settings.GEOGIT_DATASTORE and upload_session.geogit == True and import_session.tasks[0].items[0].layer.layer_type != 'RASTER'): target = create_geoserver_db_featurestore(store_type='geogit', store_name = upload_session.geogit_store) _log('setting target datastore %s %s', target.name, target.workspace.name ) import_session.tasks[0].set_target( target.name, target.workspace.name) elif (settings.OGC_SERVER['default']['OPTIONS']['DATASTORE'] != '' and import_session.tasks[0].items[0].layer.layer_type != 'RASTER'): target = create_geoserver_db_featurestore(store_type='postgis', store_name = upload_session.geogit_store) _log('setting target datastore %s %s', target.name, target.workspace.name ) import_session.tasks[0].set_target( target.name, target.workspace.name) else: target = import_session.tasks[0].target if upload_session.update_mode: _log('setting updateMode to %s', upload_session.update_mode) import_session.tasks[0].set_update_mode(upload_session.update_mode) _log('running import session') # run async if using a database import_session.commit(async) # @todo check status of import session - it may fail, but due to protocol, # this will not be reported during the commit return target
def layer_create(request, data=None, template='upload/layer_create.html'): print 'layer create' if request.method == 'POST': feature_type = json.loads(request.POST.get(u'featureType', None)) datastore = feature_type['store']['name'] store_create_geogig = json.loads(request.POST.get(u'storeCreateGeogig', 'false')) if store_create_geogig: if not feature_type['store']['name']: store_proposed = slugify(' '.join([request.user.username, feature_type['name']])) store_created = create_geoserver_db_featurestore(store_type='geogig', store_name=store_proposed) feature_type['store']['name'] = store_created.name # -- only allow creation of layers in the whitelisted datastores try: settings.ALLOWED_DATASTORE_LAYER_CREATE except AttributeError: return HttpResponse(status=FORBIDDEN, content='ALLOWED_DATASTORE_LAYER_CREATE whitelist is not defined.') if datastore not in settings.ALLOWED_DATASTORE_LAYER_CREATE: return HttpResponse(status=FORBIDDEN, content='datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.') post_request = requests.post( '{}/workspaces/{}/datastores/{}/featuretypes.json'.format(ogc_server_settings.rest, feature_type['namespace']['name'], datastore), data='{{"featureType":{}}}'.format(json.dumps(feature_type)), auth=ogc_server_settings.credentials, headers={'content-type': 'application/json'} ) if post_request.status_code == 200 or post_request.status_code == 201: # import the layer from geoserver to geonode response = gs_slurp(filter=feature_type['name'], workspace=feature_type['namespace']['name'], store=feature_type['store']['name']) if 'layers' in response and len(response['layers']) == 1 and 'name' in response['layers'][0] and response['layers'][0]['name'] == feature_type['name']: # configruer layer with time dimension as a list for now. Since importer is being refactored, # we'll change how it is done. layer = gs_catalog.get_layer(feature_type['name']) resource = layer.resource if layer: time_info = DimensionInfo( name='time', enabled=True, presentation='LIST', resolution=None, units=None, unitSymbol=None, attribute='time', end_attribute=None, strategy=None) resource.metadata = {'time': time_info} resource.catalog.save(resource) return HttpResponse(status=post_request.status_code, content=post_request.text) return HttpResponse(status=500, content='failed to configure layer') else: return HttpResponse(status=500, content='failed to add created layer from geoserver to geonode') else: return HttpResponse(status=post_request.status_code, content=post_request.text) print '---- create layer response: ', post_request.text return render_to_response(template, RequestContext(request, {}))
def import_file(self, *args, **kwargs): """ Creates a Geoserver Layer from JSON. { "name": "a new layer", "store": {"name": "store"}, "namespace": {"name": 'name'}, "attributes": {"attribute": [ {"name": "time", "binding": "org.geotools.data.postgis.PostGISDialect$XDate", "minOccurs": 0, "nillable": True}, {"name": "geometry", "binding": "com.vividsolutions.jts.geom.MultiLineString", "minOccurs":0, "nillable":True}]}, "nativeCRS": "EPSG:4326", "srs": "EPSG:4326" } """ self.completed_layers = [] configuration_options = kwargs.get('configuration_options') if not configuration_options: return # Configuration options should be a list at this point since the importer can process multiple layers in a # single import if isinstance(configuration_options, dict): configuration_options = [configuration_options] for layer in configuration_options: feature_type = layer.get('featureType') store_name = feature_type['store']['name'] store_create_geogig = layer.get(u'storeCreateGeogig', 'false') owner = layer.get('layer_owner') username = owner.username email = owner.email layer['layer_type'] = 'vector' layer['fields'] = None layer['geoserver_store'] = {'type': 'geogig'} store = layer.get('geoserver_store') if store.get('type', str).lower() == 'geogig': name = slugify(feature_type['name']) store.setdefault('branch', 'master') store.setdefault('create', 'true') store.setdefault('name', name) store['geogig_repository'] = ("geoserver://%s" % name) store_name = name feature_type['title'] = feature_type['name'] feature_type['name'] = launder(slugify(unicode(feature_type['name']))) # Without this check, the bounding box will default to 0, 0, -1, -1 if 'nativeBoundingBox' not in feature_type: feature_type['nativeBoundingBox'] = {'minx': -180, 'maxx': 180, 'miny': -90, 'maxy': 90, 'crs': 'EPSG:4326'} if store_create_geogig and store_create_geogig != 'false': store_created = create_geoserver_db_featurestore(store_type='geogig', store_name=store_name, author_name=username, author_email=email) feature_type['store']['name'] = store_created.name if store_name not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []) and '*' not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []): return HttpResponseForbidden(content='Datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.') for x in range(0, 50): post_request = requests.post( '{}/workspaces/{}/datastores/{}/featuretypes.json'.format(ogc_server_settings.rest, feature_type['namespace']['name'], store_name), data='{{"featureType":{}}}'.format(json.dumps(feature_type)), auth=ogc_server_settings.credentials, headers={'content-type': 'application/json'} ) if post_request.ok: break if 'already exists in' in post_request.content: break time.sleep(3) if post_request.ok: self.completed_layers.append([feature_type['name'], layer]) else: message = 'Unable to create layer: {0}, an unhandled exception occurred.'.format(feature_type['name']) if 'already exists in' in post_request.content: message = 'A layer named {0} already exists. Please choose another name.'.format(feature_type['name']) print post_request.content raise UploadError(message) return self.completed_layers
def create_datastore(store_name=None, store_type=None): if not store_name: store_name = ogc_server_settings.datastore_db['NAME'] return create_geoserver_db_featurestore( store_type=store_type, store_name=store_name)
def import_file(self, *args, **kwargs): """ Creates a Geoserver Layer from JSON. { "name": "a new layer", "store": {"name": "store"}, "namespace": {"name": 'name'}, "attributes": {"attribute": [ {"name": "time", "binding": "org.geotools.data.postgis.PostGISDialect$XDate", "minOccurs": 0, "nillable": True}, {"name": "geometry", "binding": "com.vividsolutions.jts.geom.MultiLineString", "minOccurs":0, "nillable":True}]}, "nativeCRS": "EPSG:4326", "srs": "EPSG:4326" } """ self.completed_layers = [] configuration_options = kwargs.get('configuration_options') if not configuration_options: return # Configuration options should be a list at this point since the importer can process multiple layers in a # single import if isinstance(configuration_options, dict): configuration_options = [configuration_options] for layer in configuration_options: feature_type = layer.get('featureType') datastore = feature_type['store']['name'] store_create_geogig = layer.get(u'storeCreateGeogig', 'false') feature_type['title'] = feature_type['name'] feature_type['name'] = launder(slugify(unicode(feature_type['name']))) # Without this check, the bounding box will default to 0, 0, -1, -1 if 'nativeBoundingBox' not in feature_type: feature_type['nativeBoundingBox'] = {'minx': -180, 'maxx': 180, 'miny': -90, 'maxy': 90, 'crs': 'EPSG:4326'} if store_create_geogig and store_create_geogig != 'false': store_created = create_geoserver_db_featurestore(store_type='geogig', store_name=feature_type['store']['name']) feature_type['store']['name'] = store_created.name if datastore not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []) and '*' not in getattr(settings, 'ALLOWED_DATASTORE_LAYER_CREATE', []): return HttpResponseForbidden(content='Datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.') post_request = requests.post( '{}/workspaces/{}/datastores/{}/featuretypes.json'.format(ogc_server_settings.rest, feature_type['namespace']['name'], datastore), data='{{"featureType":{}}}'.format(json.dumps(feature_type)), auth=ogc_server_settings.credentials, headers={'content-type': 'application/json'} ) if post_request.ok: self.completed_layers.append([feature_type['name'], layer]) else: message = 'Unable to create layer: {0}, an unhandled exception occurred.'.format(feature_type['name']) if 'already exists in' in post_request.content: message = 'A layer named {0} already exists. Please choose another name.'.format(feature_type['name']) print post_request.content raise UploadError(message) return self.completed_layers
def layer_create(request, data=None, template='upload/layer_create.html'): print 'layer create' if request.method == 'POST': feature_type = json.loads(request.POST.get(u'featureType', None)) datastore = feature_type['store']['name'] store_create_geogig = json.loads( request.POST.get(u'storeCreateGeogig', 'false')) if store_create_geogig: if not feature_type['store']['name']: store_proposed = slugify(' '.join( [request.user.username, feature_type['name']])) store_created = create_geoserver_db_featurestore( store_type='geogig', store_name=store_proposed) feature_type['store']['name'] = store_created.name # -- only allow creation of layers in the whitelisted datastores try: settings.ALLOWED_DATASTORE_LAYER_CREATE except AttributeError: return HttpResponse( status=FORBIDDEN, content= 'ALLOWED_DATASTORE_LAYER_CREATE whitelist is not defined.') if datastore not in settings.ALLOWED_DATASTORE_LAYER_CREATE: return HttpResponse( status=FORBIDDEN, content= 'datastore specified in featureType is not in the ALLOWED_DATASTORE_LAYER_CREATE whitelist.' ) post_request = requests.post( '{}/workspaces/{}/datastores/{}/featuretypes.json'.format( ogc_server_settings.rest, feature_type['namespace']['name'], datastore), data='{{"featureType":{}}}'.format(json.dumps(feature_type)), auth=ogc_server_settings.credentials, headers={'content-type': 'application/json'}) if post_request.status_code == 200 or post_request.status_code == 201: # import the layer from geoserver to geonode response = gs_slurp(filter=feature_type['name'], workspace=feature_type['namespace']['name'], store=feature_type['store']['name']) if 'layers' in response and len( response['layers'] ) == 1 and 'name' in response['layers'][0] and response['layers'][ 0]['name'] == feature_type['name']: # configruer layer with time dimension as a list for now. Since importer is being refactored, # we'll change how it is done. layer = gs_catalog.get_layer(feature_type['name']) resource = layer.resource if layer: time_info = DimensionInfo(name='time', enabled=True, presentation='LIST', resolution=None, units=None, unitSymbol=None, attribute='time', end_attribute=None, strategy=None) resource.metadata = {'time': time_info} resource.catalog.save(resource) return HttpResponse(status=post_request.status_code, content=post_request.text) return HttpResponse(status=500, content='failed to configure layer') else: return HttpResponse( status=500, content= 'failed to add created layer from geoserver to geonode') else: return HttpResponse(status=post_request.status_code, content=post_request.text) print '---- create layer response: ', post_request.text return render_to_response(template, RequestContext(request, {}))