def wfs_proxy(request, *args, **kwargs): if request.method == 'GET': endpoint = request.GET['endpoint'] typename = request.GET['typename'] user = request.session['user'] password = request.session['password'] endpoint_parsed = urlparse.urlparse(endpoint) q_dict = { 'version': '1.0.0', 'typename': typename, 'outputFormat': 'json', 'request': 'GetFeature', 'service': 'WFS', 'srsName': 'EPSG:4326', } parsed_url = urlparse.ParseResult(scheme=endpoint_parsed.scheme, netloc=endpoint_parsed.netloc, path=endpoint_parsed.path, params=None, query=urllib.urlencode(q_dict), fragment=None) LOGGER.info('Proxy to url: %s' % parsed_url.geturl()) tmpfile = download_file(parsed_url.geturl(), user=user, password=password) with open(tmpfile) as f: return HttpResponse(f.read(), content_type='application/json') return HttpResponseServerError()
def test_download_file_url(self): """Test downloading file directly using url.""" data_helper = InaSAFETestData() filename = data_helper.hazard('flood_data.geojson') hazard = file_upload(filename) wait_metadata(hazard) # download zipped layer hazard_layer_url = Analysis.get_layer_url(hazard) downloaded_file = download_file(hazard_layer_url) # check that file is downloaded self.assertTrue(os.path.exists(downloaded_file)) basename, ext = os.path.splitext(downloaded_file) # check that file has .zip extension self.assertEqual(ext, '.zip') # delete zipfile os.remove(downloaded_file) # download metadata xml file of geosafe hazard_layer_xml = reverse('geosafe:layer-metadata', kwargs={'layer_id': hazard.id}) hazard_layer_xml = urlparse.urljoin(settings.GEONODE_BASE_URL, hazard_layer_xml) downloaded_file = download_file(hazard_layer_xml) # check that file is donwloaded self.assertTrue(os.path.exists(downloaded_file)) basename, ext = os.path.splitext(downloaded_file) # check that file has .xml extension self.assertEqual(ext, '.xml') # delete xmlfile os.remove(downloaded_file) # delete layer hazard.delete()
def add_wcs_layer( endpoint, version, coverage_id, metadata_string=None, title=None, bbox=None, user=None, password=None): # build url endpoint_parsed = urlparse.urlparse(endpoint) q_dict = { 'version': version, 'coverageid': coverage_id, 'format': 'image/tiff', 'request': 'GetCoverage', 'service': 'WCS', 'crs': 'EPSG:4326', } if bbox: q_dict['bbox'] = ','.join(bbox) parsed_url = urlparse.ParseResult( scheme=endpoint_parsed.scheme, netloc=endpoint_parsed.netloc, path=endpoint_parsed.path, params=None, query=urllib.urlencode(q_dict), fragment=None ) tmpfile = download_file(parsed_url.geturl(), user=user, password=password) shutil.move(tmpfile, tmpfile + '.tif') metadata_file = '%s.xml' % tmpfile tmpfile += '.tif' # get metadata file if metadata_string: if not isinstance(metadata_string, unicode): metadata_string = unicode(metadata_string, 'utf-8') metadata_string = cleanup_metadata(metadata_string) with io.open(metadata_file, mode='w', encoding='utf-8') as f: f.write(metadata_string) saved_layer = None with open(tmpfile) as f: saved_layer = file_upload(tmpfile, overwrite=True) saved_layer.set_default_permissions() saved_layer.title = title or coverage_id saved_layer.save() try: os.remove(tmpfile) os.remove(metadata_file) except: pass return saved_layer
def process_impact_result(self, impact_result, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_result: A dictionary of output's layer key and Uri with status and message. :type impact_result: dict :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) success = False report_success = False impact_url = None impact_path = None if impact_result['status'] == RESULT_SUCCESS: impact_url = ( impact_result['output'].get('impact_analysis') or impact_result['output'].get('hazard_aggregation_summary')) analysis_summary_url = ( impact_result['output'].get('analysis_summary')) custom_template_path = prepare_custom_template(analysis, impact_url) layer_order = prepare_context_layer_order(analysis, impact_url) # generate report when analysis has ran successfully result = generate_report.delay( impact_url, # If it is None, it will use default headless template custom_report_template_uri=custom_template_path, custom_layer_order=layer_order, locale=analysis.language_code) retries = 10 for r in range(retries): try: with allow_join_result(): report_metadata = result.get().get('output', {}) break except BaseException as e: if result.state == 'FAILURE': raise e LOGGER.exception(e) time.sleep(5) result = AsyncResult(result.id) if r >= retries - 1: # Generate report has failed. # We need to reraise the error so we know something is # wrong raise e for product_key, products in report_metadata.iteritems(): for report_key, report_url in products.iteritems(): report_url = download_file(report_url, direct_access=True) report_metadata[product_key][report_key] = report_url # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact layer path impact_path = download_file(impact_url, direct_access=True) dir_name = os.path.dirname(impact_path) is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip' if is_zipfile: # Extract the layer first with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in cov_exts + vec_exts: # process this in the for loop to make sure it # works only when we found the layer success = process_impact_layer( analysis, dir_name, basename, name) report_success = process_impact_report( analysis, report_metadata) break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass else: # It means it is accessing an shp or tif directly analysis_summary_filename = os.path.basename(analysis_summary_url) impact_filename = os.path.basename(impact_path) impact_basename, ext = os.path.splitext(impact_filename) success = process_impact_layer( analysis, dir_name, impact_basename, impact_filename, analysis_summary_filename) report_success = process_impact_report(analysis, report_metadata) # cleanup for name in os.listdir(dir_name): filepath = os.path.join(dir_name, name) is_file = os.path.isfile(filepath) should_delete = name.split('.')[0] == impact_basename if is_file and should_delete: try: os.remove(filepath) except BaseException: pass # cleanup try: os.remove(impact_path) except BaseException: pass if not success: LOGGER.info('No impact layer found in {0}'.format(impact_url)) if not report_success: LOGGER.info('No impact report generated.') send_analysis_result_email(analysis) return success
def test_download_file_path(self): """Test that download_file returns correctly. Will test against several file:// scheme """ cases = [ { # Check regular file:// scheme case 'input': 'file:///foo/bar.zip', 'output': '/foo/bar.zip' }, { # Check that relative file:// scheme case 'input': 'file://foo/bar.zip', 'output': '/bar.zip' }, { # Check with + sign 'input': 'file:///foo+bar/file+name+long.zip', 'output': '/foo bar/file name long.zip' }, { # Check urlencoded 'input': 'file:///foo%20bar%2Flong%20file%2Cname.zip', 'output': '/foo bar/long file,name.zip' }, { # Check urlencoded and utf-8 encode 'input': 'file:///movies%2F' '%E5%90%9B%E3%81%AE%E5%90%8D%E3%81%AF.mkv', 'output': u'/movies/君の名は.mkv' }, { # Check regular path 'input': '/foo/bar.xml', 'output': '/foo/bar.xml' }, { # Check do not decode regular path 'input': '/foo/file+name.shp', 'output': '/foo/file+name.shp' }, { # Check do not decode regular path 'input': '/foo/file%20name.shp', 'output': '/foo/file%20name.shp' }, { # Check do not decode regular path 'input': u'/movies/君の名は.mkv', 'output': u'/movies/君の名は.mkv' } ] for case in cases: message = 'The following case has failed: {case}'.format(case=case) self.assertEqual(case.get('output'), download_file(case.get('input'), direct_access=True), msg=message) # Check that download file will generate temporary file data_helper = InaSAFETestData() filename = data_helper.hazard('flood_data.geojson') hazard = file_upload(filename) """:type: geonode.layers.models.Layer""" wait_metadata(hazard) hazard_base_file, _ = hazard.get_base_file() hazard_path = hazard_base_file.file.path # Check file exists self.assertTrue(os.path.exists(hazard_path)) downloaded_file = download_file(hazard_path, direct_access=False) # Should be a different filename self.assertNotEqual(hazard_path, downloaded_file) # Check file exists self.assertTrue(os.path.exists(downloaded_file)) os.remove(downloaded_file) hazard.delete()
def add_wfs_layer(endpoint, version, typename, metadata_string=None, title=None, bbox=None, user=None, password=None): endpoint_parsed = urlparse.urlparse(endpoint) q_dict = { 'version': version, 'typename': typename, 'outputFormat': 'shape-zip', 'request': 'GetFeature', 'service': 'WFS', 'srsName': 'EPSG:4326', } if bbox: q_dict['bbox'] = ','.join(bbox) parsed_url = urlparse.ParseResult(scheme=endpoint_parsed.scheme, netloc=endpoint_parsed.netloc, path=endpoint_parsed.path, params=None, query=urllib.urlencode(q_dict), fragment=None) tmpfile = download_file(parsed_url.geturl(), user=user, password=password) # args = [ # 'ogr2ogr', # '-nlt POLYGON', # '-skipfailures', # '%s.shp' % tmpfile, # tmpfile, # 'OGRGeoJSON' # ] # # retval = subprocess.call(args) # # get metadata file # if metadata_string: # if not isinstance(metadata_string, unicode): # metadata_string = unicode(metadata_string, 'utf-8') # metadata_file = '%s.xml' % tmpfile # with io.open(metadata_file, mode='w', encoding='utf-8') as f: # f.write(metadata_string) # # saved_layer = None # if retval == 0: # saved_layer = file_upload( # '%s.shp' % tmpfile, # overwrite=True) # saved_layer.set_default_permissions() # saved_layer.title = title or typename # saved_layer.save() # # # cleanup # dir_name = os.path.dirname(tmpfile) # for root, dirs, files in os.walk(dir_name): # for f in files: # if tmpfile in f: # try: # os.remove(os.path.join(root, f)) # except: # pass dir_name = os.path.dirname(tmpfile) saved_layer = None metadata_file = None with ZipFile(tmpfile) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if '.shp' in ext: # get metadata file if metadata_string: if not isinstance(metadata_string, unicode): metadata_string = unicode(metadata_string, 'utf-8') metadata_file = '%s.xml' % basename metadata_file = os.path.join(dir_name, metadata_file) metadata_string = cleanup_metadata(metadata_string) with io.open(metadata_file, mode='w', encoding='utf-8') as f: f.write(metadata_string) # process shapefile layer saved_layer = file_upload(os.path.join(dir_name, name), overwrite=True) saved_layer.set_default_permissions() saved_layer.title = title or typename saved_layer.save() break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass if metadata_file: try: os.remove(metadata_file) except BaseException: pass # cleanup try: os.remove(tmpfile) except BaseException: pass return saved_layer
def process_impact_result(self, impact_result, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_result: A dictionary of output's layer key and Uri with status and message. :type impact_result: dict :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) analysis.task_id = self.request.id analysis.save() success = False report_success = False impact_url = None impact_path = None if impact_result['status'] == RESULT_SUCCESS: impact_url = ( impact_result['output'].get('impact_analysis') or impact_result['output'].get('hazard_aggregation_summary')) analysis_summary_url = ( impact_result['output'].get('analysis_summary')) # generate report when analysis has ran successfully async = generate_report.delay(impact_url) with allow_join_result(): report_metadata = async .get().get('output', {}) for product_key, products in report_metadata.iteritems(): for report_key, report_url in products.iteritems(): report_url = download_file(report_url, direct_access=True) report_metadata[product_key][report_key] = report_url # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact layer path impact_path = download_file(impact_url, direct_access=True) dir_name = os.path.dirname(impact_path) is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip' if is_zipfile: # Extract the layer first with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in cov_exts + vec_exts: # process this in the for loop to make sure it # works only when we found the layer success = process_impact_layer(analysis, dir_name, basename, name) report_success = process_impact_report( analysis, report_metadata) break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass else: # It means it is accessing an shp or tif directly analysis_summary_filename = os.path.basename(analysis_summary_url) impact_filename = os.path.basename(impact_path) impact_basename, ext = os.path.splitext(impact_filename) success = process_impact_layer(analysis, dir_name, impact_basename, impact_filename, analysis_summary_filename) report_success = process_impact_report(analysis, report_metadata) # cleanup for name in os.listdir(dir_name): filepath = os.path.join(dir_name, name) is_file = os.path.isfile(filepath) should_delete = name.split('.')[0] == impact_basename if is_file and should_delete: try: os.remove(filepath) except BaseException: pass # cleanup try: os.remove(impact_path) except BaseException: pass if not success: LOGGER.info('No impact layer found in {0}'.format(impact_url)) if not report_success: LOGGER.info('No impact report generated.') return success
def process_impact_result(self, impact_url, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_url: impact url returned from analysis :type impact_url: str :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) analysis.task_id = self.request.id analysis.save() # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact zip impact_path = download_file(impact_url) dir_name = os.path.dirname(impact_path) success = False with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in ['.shp', '.tif']: # process this in the for loop to make sure it works only # when we found the layer saved_layer = file_upload(os.path.join(dir_name, name), overwrite=True) saved_layer.set_default_permissions() if analysis.user_title: layer_name = analysis.user_title else: layer_name = analysis.get_default_impact_title() saved_layer.title = layer_name saved_layer.save() current_impact = None if analysis.impact_layer: current_impact = analysis.impact_layer analysis.impact_layer = saved_layer # check map report and table report_map_path = os.path.join(dir_name, '%s.pdf' % basename) if os.path.exists(report_map_path): analysis.assign_report_map(report_map_path) report_table_path = os.path.join(dir_name, '%s_table.pdf' % basename) if os.path.exists(report_table_path): analysis.assign_report_table(report_table_path) analysis.task_id = process_impact_result.request.id analysis.task_state = 'SUCCESS' analysis.save() if current_impact: current_impact.delete() success = True break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except: pass # cleanup try: os.remove(impact_path) except: pass if not success: LOGGER.info('No impact layer found in %s' % impact_url) return success
def process_impact_result(self, impact_url, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_url: impact url returned from analysis :type impact_url: str :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) analysis.task_id = self.request.id analysis.save() # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact layer path impact_path = download_file(impact_url, direct_access=True) dir_name = os.path.dirname(impact_path) success = False is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip' if is_zipfile: # Extract the layer first with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in ['.shp', '.tif']: # process this in the for loop to make sure it works only # when we found the layer success = process_impact_layer( analysis, basename, dir_name, name) break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass else: # It means it is accessing an shp or tif directly filename = os.path.basename(impact_path) basename, ext = os.path.splitext(filename) success = process_impact_layer(analysis, basename, dir_name, filename) # cleanup for name in os.listdir(dir_name): filepath = os.path.join(dir_name, name) is_file = os.path.isfile(filepath) should_delete = name.split('.')[0] == basename if is_file and should_delete: try: os.remove(filepath) except BaseException: pass # cleanup try: os.remove(impact_path) except BaseException: pass if not success: LOGGER.info('No impact layer found in %s' % impact_url) return success