def prepare_custom_template(analysis, impact_url): """This helper method will prepare custom template. This template will be included in report generations. :param analysis: Analysis object to prepare :type analysis: Analysis :param impact_url: The impact URI returned by Headless :type impact_url: basestring :return: The location of custom template path, relative to Headless service :rtype: basestring """ # define the location of qgis template file custom_template_path = None if analysis.custom_template: # resolve headless impact layer directory on django environment filename = os.path.basename(analysis.custom_template) dirname = os.path.dirname(impact_url) # template path from headless service headless_template_path = os.path.join(dirname, filename) # template path in GeoSAFE template_path = get_impact_path(headless_template_path) # each analysis path is unique, so we can just copy/overwrite # the template shutil.copy(analysis.custom_template, template_path) # pass on template path according to headless service custom_template_path = headless_template_path return custom_template_path
def test_get_impact_path(self): """Test return impact file path if using direct file access.""" impact_url = 'http://inasafe-output/output/200/tmp1234.zip' converted_impact_path = get_impact_path(impact_url) # Use direct access by default self.assertEqual(converted_impact_path, '/home/geosafe/impact_layers/200/tmp1234.zip') geosafe_impact_output_dir = settings.GEOSAFE_IMPACT_OUTPUT_DIRECTORY settings.set('GEOSAFE_IMPACT_OUTPUT_DIRECTORY', None) converted_impact_path = get_impact_path(impact_url) # Doesn't change anything if not configured self.assertEqual(converted_impact_path, impact_url) settings.set('GEOSAFE_IMPACT_OUTPUT_DIRECTORY', geosafe_impact_output_dir)
def process_impact_report(analysis, report_metadata): """Internal method to process impact report. :param analysis: Analysis object :type analysis: Analysis :param report_metadata: Impact report metadata :type report_metadata: dict :return: True if success """ success = False try: # upload using document upload form post request # TODO: find out how to upload document using post request assign_report = { 'impact-report-pdf': analysis.assign_report_table } if analysis.custom_template: # If we provide custom_template, search custom template key assign_report['map-report'] = analysis.assign_report_map else: # If not, pick default headless report assign_report['inasafe-map-report-portrait'] = \ analysis.assign_report_map # List of tags of PDF report product pdf_product_tag = report_metadata['pdf_product_tag'] for metadata_key in pdf_product_tag.keys(): for assign_key, assign_method in assign_report.iteritems(): # If the key we search doesn't exists in metadata, # just continue if assign_key not in metadata_key: continue # if it is exists, check the file is exists too report_path = pdf_product_tag[metadata_key] # convert to GeoSAFE path location report_path = get_impact_path(report_path) if os.path.exists(report_path): # save the path location assign_method(report_path) # Avoid race conditions analysis.save(update_fields=['report_map', 'report_table']) # reference to impact layer # TODO: find out how to upload document using post request first # delete all possible temporary styles being used to generate report layer_list = [ analysis.hazard_layer, analysis.exposure_layer ] if analysis.aggregation_layer: layer_list.append(analysis.aggregation_layer) for l in layer_list: l.qgis_layer.remove_qml_file_style() success = True except Exception as e: LOGGER.debug(e) pass return success
def process_impact_result(self, impact_result, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_result: A dictionary of output's layer key and Uri with status and message. :type impact_result: dict :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) success = False report_success = False impact_url = None impact_path = None if impact_result['status'] == RESULT_SUCCESS: impact_url = ( impact_result['output'].get('impact_analysis') or impact_result['output'].get('hazard_aggregation_summary')) analysis_summary_url = ( impact_result['output'].get('analysis_summary')) custom_template_path = prepare_custom_template(analysis, impact_url) layer_order = prepare_context_layer_order(analysis, impact_url) # generate report when analysis has ran successfully result = generate_report.delay( impact_url, # If it is None, it will use default headless template custom_report_template_uri=custom_template_path, custom_layer_order=layer_order, locale=analysis.language_code) retries = 10 for r in range(retries): try: with allow_join_result(): report_metadata = result.get().get('output', {}) break except BaseException as e: if result.state == 'FAILURE': raise e LOGGER.exception(e) time.sleep(5) result = AsyncResult(result.id) if r >= retries - 1: # Generate report has failed. # We need to reraise the error so we know something is # wrong raise e for product_key, products in report_metadata.iteritems(): for report_key, report_url in products.iteritems(): report_url = download_file(report_url, direct_access=True) report_metadata[product_key][report_key] = report_url # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact layer path impact_path = download_file(impact_url, direct_access=True) dir_name = os.path.dirname(impact_path) is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip' if is_zipfile: # Extract the layer first with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in cov_exts + vec_exts: # process this in the for loop to make sure it # works only when we found the layer success = process_impact_layer( analysis, dir_name, basename, name) report_success = process_impact_report( analysis, report_metadata) break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass else: # It means it is accessing an shp or tif directly analysis_summary_filename = os.path.basename(analysis_summary_url) impact_filename = os.path.basename(impact_path) impact_basename, ext = os.path.splitext(impact_filename) success = process_impact_layer( analysis, dir_name, impact_basename, impact_filename, analysis_summary_filename) report_success = process_impact_report(analysis, report_metadata) # cleanup for name in os.listdir(dir_name): filepath = os.path.join(dir_name, name) is_file = os.path.isfile(filepath) should_delete = name.split('.')[0] == impact_basename if is_file and should_delete: try: os.remove(filepath) except BaseException: pass # cleanup try: os.remove(impact_path) except BaseException: pass if not success: LOGGER.info('No impact layer found in {0}'.format(impact_url)) if not report_success: LOGGER.info('No impact report generated.') send_analysis_result_email(analysis) return success
def process_impact_result(self, impact_result, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_result: A dictionary of output's layer key and Uri with status and message. :type impact_result: dict :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) analysis.task_id = self.request.id analysis.save() success = False report_success = False impact_url = None impact_path = None if impact_result['status'] == RESULT_SUCCESS: impact_url = ( impact_result['output'].get('impact_analysis') or impact_result['output'].get('hazard_aggregation_summary')) analysis_summary_url = ( impact_result['output'].get('analysis_summary')) # generate report when analysis has ran successfully async = generate_report.delay(impact_url) with allow_join_result(): report_metadata = async .get().get('output', {}) for product_key, products in report_metadata.iteritems(): for report_key, report_url in products.iteritems(): report_url = download_file(report_url, direct_access=True) report_metadata[product_key][report_key] = report_url # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact layer path impact_path = download_file(impact_url, direct_access=True) dir_name = os.path.dirname(impact_path) is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip' if is_zipfile: # Extract the layer first with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in cov_exts + vec_exts: # process this in the for loop to make sure it # works only when we found the layer success = process_impact_layer(analysis, dir_name, basename, name) report_success = process_impact_report( analysis, report_metadata) break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass else: # It means it is accessing an shp or tif directly analysis_summary_filename = os.path.basename(analysis_summary_url) impact_filename = os.path.basename(impact_path) impact_basename, ext = os.path.splitext(impact_filename) success = process_impact_layer(analysis, dir_name, impact_basename, impact_filename, analysis_summary_filename) report_success = process_impact_report(analysis, report_metadata) # cleanup for name in os.listdir(dir_name): filepath = os.path.join(dir_name, name) is_file = os.path.isfile(filepath) should_delete = name.split('.')[0] == impact_basename if is_file and should_delete: try: os.remove(filepath) except BaseException: pass # cleanup try: os.remove(impact_path) except BaseException: pass if not success: LOGGER.info('No impact layer found in {0}'.format(impact_url)) if not report_success: LOGGER.info('No impact report generated.') return success
def process_impact_result(self, impact_url, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_url: impact url returned from analysis :type impact_url: str :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) analysis.task_id = self.request.id analysis.save() # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact zip impact_path = download_file(impact_url) dir_name = os.path.dirname(impact_path) success = False with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in ['.shp', '.tif']: # process this in the for loop to make sure it works only # when we found the layer saved_layer = file_upload(os.path.join(dir_name, name), overwrite=True) saved_layer.set_default_permissions() if analysis.user_title: layer_name = analysis.user_title else: layer_name = analysis.get_default_impact_title() saved_layer.title = layer_name saved_layer.save() current_impact = None if analysis.impact_layer: current_impact = analysis.impact_layer analysis.impact_layer = saved_layer # check map report and table report_map_path = os.path.join(dir_name, '%s.pdf' % basename) if os.path.exists(report_map_path): analysis.assign_report_map(report_map_path) report_table_path = os.path.join(dir_name, '%s_table.pdf' % basename) if os.path.exists(report_table_path): analysis.assign_report_table(report_table_path) analysis.task_id = process_impact_result.request.id analysis.task_state = 'SUCCESS' analysis.save() if current_impact: current_impact.delete() success = True break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except: pass # cleanup try: os.remove(impact_path) except: pass if not success: LOGGER.info('No impact layer found in %s' % impact_url) return success
def process_impact_result(self, impact_url, analysis_id): """Extract impact analysis after running it via InaSAFE-Headless celery :param self: Task instance :type self: celery.task.Task :param impact_url: impact url returned from analysis :type impact_url: str :param analysis_id: analysis id of the object :type analysis_id: int :return: True if success :rtype: bool """ # Track the current task_id analysis = Analysis.objects.get(id=analysis_id) analysis.task_id = self.request.id analysis.save() # decide if we are using direct access or not impact_url = get_impact_path(impact_url) # download impact layer path impact_path = download_file(impact_url, direct_access=True) dir_name = os.path.dirname(impact_path) success = False is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip' if is_zipfile: # Extract the layer first with ZipFile(impact_path) as zf: zf.extractall(path=dir_name) for name in zf.namelist(): basename, ext = os.path.splitext(name) if ext in ['.shp', '.tif']: # process this in the for loop to make sure it works only # when we found the layer success = process_impact_layer( analysis, basename, dir_name, name) break # cleanup for name in zf.namelist(): filepath = os.path.join(dir_name, name) try: os.remove(filepath) except BaseException: pass else: # It means it is accessing an shp or tif directly filename = os.path.basename(impact_path) basename, ext = os.path.splitext(filename) success = process_impact_layer(analysis, basename, dir_name, filename) # cleanup for name in os.listdir(dir_name): filepath = os.path.join(dir_name, name) is_file = os.path.isfile(filepath) should_delete = name.split('.')[0] == basename if is_file and should_delete: try: os.remove(filepath) except BaseException: pass # cleanup try: os.remove(impact_path) except BaseException: pass if not success: LOGGER.info('No impact layer found in %s' % impact_url) return success