def append_feature_chunks(features, wfst_insert_template,get_features_request): summary = None wfs_transaction_payload = wfst_insert_template.format(features=''.join(features), workspace='geonode', workspace_uri='http://www.geonode.org/') insert_features_request = requests.post( '{}/wfs/WfsDispatcher'.format(ogc_server_settings.public_url), auth=ogc_server_settings.credentials, headers={'Content-Type': 'application/xml'}, data=wfs_transaction_payload ) if has_exception(insert_features_request.content) is False: print_exception(insert_features_request.content) else: summary = parse_wfst_response(insert_features_request.content) return summary
def append_feature_chunks(features, wfst_insert_template, get_features_request): summary = None wfs_transaction_payload = wfst_insert_template.format( features=''.join(features), workspace='geonode', workspace_uri='http://www.geonode.org/') insert_features_request = requests.post( '{}/wfs/WfsDispatcher'.format(ogc_server_settings.public_url), auth=ogc_server_settings.credentials, headers={'Content-Type': 'application/xml'}, data=wfs_transaction_payload) if has_exception(insert_features_request.content) is False: print_exception(insert_features_request.content) else: summary = parse_wfst_response(insert_features_request.content) return summary
def layer_append_minimal(source, target, request_cookies): """ The main layer_append logic that can run outside of a request. """ source = 'geonode:' + source def chunk_list(list, chunk_size): """Yield successive chunk_size chunks from list.""" for i in xrange(0, len(list), chunk_size): yield list[i:i+chunk_size] # TODO: use the provided column to decide which features should be updated and which should be created # join_on_attribute = json.loads(request.POST.get(u'joinOnAttributeName', 'false')) get_features_request = requests.post( '{}/wfs?service=wfs&version=2.0.0&request=GetFeature&typeNames={}'.format(ogc_server_settings.public_url, source), auth=ogc_server_settings.credentials ) if has_exception(get_features_request.content): raise AssertionError('Failed to get features from source layer: {0}'.format(source)) # the response to getfeatures will look like the following. We want everything between first <wfs:member> and last </wfs:member> # <wfs:FeatureCollection ...> # <wfs:member> # <geonode:a3 gml:id="a3.4"> # <geonode:geometry> # <gml:Point srsDimension="2" srsName="urn:ogc:def:crs:EPSG::4326"> # <gml:pos>14.101186235070415 -87.19960869178765</gml:pos> # </gml:Point> # </geonode:geometry> # </geonode:a3> # </wfs:member> # ... # <wfs:member> # ... # </wfs:member> # </wfs:FeatureCollection> # Create the xml containing all the features that need to be posted. Need to get the features form the source # layer, update them so that they get posted to the destination layer xml = etree.XML(get_features_request.content) tree = etree.ElementTree(xml) root = tree.getroot() for ns in root.nsmap: xpath_ns = etree.FunctionNamespace(root.nsmap[ns]) xpath_ns.prefix = ns members = tree.xpath('//wfs:FeatureCollection/wfs:member') members_str = [] for m in members: # replace the tag <workspace>:<layer_destination> with <workspace>:<layer_source> tokens = target.split(':') layername_element = m.find(source, root.nsmap) layername_element.tag = '{' + root.nsmap[tokens[0]] + '}' + tokens[1] members_str.append(etree.tostring(m)) # divide the features (members_str) into chunks so that we can have a progress indicator features_per_chunk = 750 features_chunks = chunk_list(members_str, features_per_chunk) # example of transactions can be found at: # https://github.com/highsource/ogc-schemas/tree/2.0.0/schemas/src/main/resources/ogc/wfs/2.0/examples wfst_insert_template = ' '.join(( '<?xml version="1.0" encoding="utf-8"?>', '<wfs:Transaction', 'service="WFS"', 'version="2.0.0"', 'xmlns:{workspace}="{workspace_uri}"', 'xmlns:gml="http://www.opengis.net/gml/3.2"', 'xmlns:wfs="http://www.opengis.net/wfs/2.0"', 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"', 'handle="{handle}"', 'xsi:schemaLocation="', 'http://www.opengis.net/wfs/2.0', 'http://schemas.opengis.net/wfs/2.0/wfs.xsd', 'http://www.opengis.net/gml/3.2', 'http://schemas.opengis.net/gml/3.2.1/gml.xsd">', '<wfs:Insert>', '{features}', '</wfs:Insert>', '</wfs:Transaction>' )) summary_aggregate = [] summary = None for features in features_chunks: handle = 'added {0} features to {1} via append'.format(len(features), target) wfs_transaction_payload = wfst_insert_template.format(features=''.join(features), workspace='geonode', workspace_uri='http://www.geonode.org/', handle=handle) insert_features_request = requests.post( '{}/wfs/WfsDispatcher'.format(ogc_server_settings.public_url), cookies=request_cookies, headers={'Content-Type': 'application/xml'}, data=wfs_transaction_payload ) if has_exception(insert_features_request.content) is False: print_exception(insert_features_request.content) else: summary = parse_wfst_response(insert_features_request.content) summary_aggregate.append(summary) #insert_tasks = group(tasks.append_feature_chunks.subtask((features,wfst_insert_template,get_features_request,target)) for features in features_chunks) #results = insert_tasks.apply_async() #insert_summary = results.join() return summary_aggregate
def layer_append_minimal(source, target, request_cookies): """ The main layer_append logic that can run outside of a request. """ source = 'geonode:' + source def chunk_list(list, chunk_size): """Yield successive chunk_size chunks from list.""" for i in xrange(0, len(list), chunk_size): yield list[i:i+chunk_size] # TODO: use the provided column to decide which features should be updated and which should be created # join_on_attribute = json.loads(request.POST.get(u'joinOnAttributeName', 'false')) get_features_request = requests.post( '{}/wfs?service=wfs&version=2.0.0&request=GetFeature&typeNames={}'.format(ogc_server_settings.public_url, source), auth=ogc_server_settings.credentials ) if has_exception(get_features_request.content): raise AssertionError( 'Failed to get features from source layer: {0}'.format(source)) # the response to getfeatures will look like the following. We want everything between first <wfs:member> and last </wfs:member> # <wfs:FeatureCollection ...> # <wfs:member> # <geonode:a3 gml:id="a3.4"> # <geonode:geometry> # <gml:Point srsDimension="2" srsName="urn:ogc:def:crs:EPSG::4326"> # <gml:pos>14.101186235070415 -87.19960869178765</gml:pos> # </gml:Point> # </geonode:geometry> # </geonode:a3> # </wfs:member> # ... # <wfs:member> # ... # </wfs:member> # </wfs:FeatureCollection> # Create the xml containing all the features that need to be posted. Need to get the features form the source # layer, update them so that they get posted to the destination layer xml = etree.XML(get_features_request.content) tree = etree.ElementTree(xml) root = tree.getroot() for ns in root.nsmap: xpath_ns = etree.FunctionNamespace(root.nsmap[ns]) xpath_ns.prefix = ns members = tree.xpath('//wfs:FeatureCollection/wfs:member') members_str = [] for m in members: # replace the tag <workspace>:<layer_destination> with <workspace>:<layer_source> tokens = target.split(':') layername_element = m.find(source, root.nsmap) layername_element.tag = '{' + root.nsmap[tokens[0]] + '}' + tokens[1] members_str.append(etree.tostring(m)) # divide the features (members_str) into chunks so that we can have a progress indicator features_per_chunk = 750 features_chunks = chunk_list(members_str, features_per_chunk) # example of transactions can be found at: # https://github.com/highsource/ogc-schemas/tree/2.0.0/schemas/src/main/resources/ogc/wfs/2.0/examples wfst_insert_template = ' '.join(( '<?xml version="1.0" encoding="utf-8"?>', '<wfs:Transaction', 'service="WFS"', 'version="2.0.0"', 'xmlns:{workspace}="{workspace_uri}"', 'xmlns:gml="http://www.opengis.net/gml/3.2"', 'xmlns:wfs="http://www.opengis.net/wfs/2.0"', 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"', 'handle="{handle}"', 'xsi:schemaLocation="', 'http://www.opengis.net/wfs/2.0', 'http://schemas.opengis.net/wfs/2.0/wfs.xsd', 'http://www.opengis.net/gml/3.2', 'http://schemas.opengis.net/gml/3.2.1/gml.xsd">', '<wfs:Insert>', '{features}', '</wfs:Insert>', '</wfs:Transaction>' )) summary_aggregate = [] summary = None for features in features_chunks: handle = 'added {0} features to {1} via append'.format( len(features), target) wfs_transaction_payload = wfst_insert_template.format(features=''.join(features), workspace='geonode', workspace_uri='http://www.geonode.org/', handle=handle) insert_features_request = requests.post( '{}/wfs/WfsDispatcher'.format(ogc_server_settings.public_url), cookies=request_cookies, headers={'Content-Type': 'application/xml'}, data=wfs_transaction_payload ) if has_exception(insert_features_request.content) is False: print_exception(insert_features_request.content) else: summary = parse_wfst_response(insert_features_request.content) summary_aggregate.append(summary) return summary_aggregate