def prepare_custom_template(analysis, impact_url):
"""This helper method will prepare custom template.
This template will be included in report generations.
:param analysis: Analysis object to prepare
:type analysis: Analysis
:param impact_url: The impact URI returned by Headless
:type impact_url: basestring
:return: The location of custom template path, relative to
Headless service
:rtype: basestring
"""
# define the location of qgis template file
custom_template_path = None
if analysis.custom_template:
# resolve headless impact layer directory on django environment
filename = os.path.basename(analysis.custom_template)
dirname = os.path.dirname(impact_url)
# template path from headless service
headless_template_path = os.path.join(dirname, filename)
# template path in GeoSAFE
template_path = get_impact_path(headless_template_path)
# each analysis path is unique, so we can just copy/overwrite
# the template
shutil.copy(analysis.custom_template, template_path)
# pass on template path according to headless service
custom_template_path = headless_template_path
return custom_template_path
def test_get_impact_path(self):
"""Test return impact file path if using direct file access."""
impact_url = 'http://inasafe-output/output/200/tmp1234.zip'
converted_impact_path = get_impact_path(impact_url)
# Use direct access by default
self.assertEqual(converted_impact_path,
'/home/geosafe/impact_layers/200/tmp1234.zip')
geosafe_impact_output_dir = settings.GEOSAFE_IMPACT_OUTPUT_DIRECTORY
settings.set('GEOSAFE_IMPACT_OUTPUT_DIRECTORY', None)
converted_impact_path = get_impact_path(impact_url)
# Doesn't change anything if not configured
self.assertEqual(converted_impact_path, impact_url)
settings.set('GEOSAFE_IMPACT_OUTPUT_DIRECTORY',
geosafe_impact_output_dir)
def process_impact_report(analysis, report_metadata):
"""Internal method to process impact report.
:param analysis: Analysis object
:type analysis: Analysis
:param report_metadata: Impact report metadata
:type report_metadata: dict
:return: True if success
"""
success = False
try:
# upload using document upload form post request
# TODO: find out how to upload document using post request
assign_report = {
'impact-report-pdf': analysis.assign_report_table
}
if analysis.custom_template:
# If we provide custom_template, search custom template key
assign_report['map-report'] = analysis.assign_report_map
else:
# If not, pick default headless report
assign_report['inasafe-map-report-portrait'] = \
analysis.assign_report_map
# List of tags of PDF report product
pdf_product_tag = report_metadata['pdf_product_tag']
for metadata_key in pdf_product_tag.keys():
for assign_key, assign_method in assign_report.iteritems():
# If the key we search doesn't exists in metadata,
# just continue
if assign_key not in metadata_key:
continue
# if it is exists, check the file is exists too
report_path = pdf_product_tag[metadata_key]
# convert to GeoSAFE path location
report_path = get_impact_path(report_path)
if os.path.exists(report_path):
# save the path location
assign_method(report_path)
# Avoid race conditions
analysis.save(update_fields=['report_map', 'report_table'])
# reference to impact layer
# TODO: find out how to upload document using post request first
# delete all possible temporary styles being used to generate report
layer_list = [
analysis.hazard_layer,
analysis.exposure_layer
]
if analysis.aggregation_layer:
layer_list.append(analysis.aggregation_layer)
for l in layer_list:
l.qgis_layer.remove_qml_file_style()
success = True
except Exception as e:
LOGGER.debug(e)
pass
return success
def process_impact_result(self, impact_result, analysis_id):
"""Extract impact analysis after running it via InaSAFE-Headless celery
:param self: Task instance
:type self: celery.task.Task
:param impact_result: A dictionary of output's layer key and Uri with
status and message.
:type impact_result: dict
:param analysis_id: analysis id of the object
:type analysis_id: int
:return: True if success
:rtype: bool
"""
# Track the current task_id
analysis = Analysis.objects.get(id=analysis_id)
success = False
report_success = False
impact_url = None
impact_path = None
if impact_result['status'] == RESULT_SUCCESS:
impact_url = (
impact_result['output'].get('impact_analysis') or
impact_result['output'].get('hazard_aggregation_summary'))
analysis_summary_url = (
impact_result['output'].get('analysis_summary'))
custom_template_path = prepare_custom_template(analysis, impact_url)
layer_order = prepare_context_layer_order(analysis, impact_url)
# generate report when analysis has ran successfully
result = generate_report.delay(
impact_url,
# If it is None, it will use default headless template
custom_report_template_uri=custom_template_path,
custom_layer_order=layer_order,
locale=analysis.language_code)
retries = 10
for r in range(retries):
try:
with allow_join_result():
report_metadata = result.get().get('output', {})
break
except BaseException as e:
if result.state == 'FAILURE':
raise e
LOGGER.exception(e)
time.sleep(5)
result = AsyncResult(result.id)
if r >= retries - 1:
# Generate report has failed.
# We need to reraise the error so we know something is
# wrong
raise e
for product_key, products in report_metadata.iteritems():
for report_key, report_url in products.iteritems():
report_url = download_file(report_url, direct_access=True)
report_metadata[product_key][report_key] = report_url
# decide if we are using direct access or not
impact_url = get_impact_path(impact_url)
# download impact layer path
impact_path = download_file(impact_url, direct_access=True)
dir_name = os.path.dirname(impact_path)
is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip'
if is_zipfile:
# Extract the layer first
with ZipFile(impact_path) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if ext in cov_exts + vec_exts:
# process this in the for loop to make sure it
# works only when we found the layer
success = process_impact_layer(
analysis, dir_name, basename, name)
report_success = process_impact_report(
analysis, report_metadata)
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except BaseException:
pass
else:
# It means it is accessing an shp or tif directly
analysis_summary_filename = os.path.basename(analysis_summary_url)
impact_filename = os.path.basename(impact_path)
impact_basename, ext = os.path.splitext(impact_filename)
success = process_impact_layer(
analysis, dir_name, impact_basename,
impact_filename, analysis_summary_filename)
report_success = process_impact_report(analysis, report_metadata)
# cleanup
for name in os.listdir(dir_name):
filepath = os.path.join(dir_name, name)
is_file = os.path.isfile(filepath)
should_delete = name.split('.')[0] == impact_basename
if is_file and should_delete:
try:
os.remove(filepath)
except BaseException:
pass
# cleanup
try:
os.remove(impact_path)
except BaseException:
pass
if not success:
LOGGER.info('No impact layer found in {0}'.format(impact_url))
if not report_success:
LOGGER.info('No impact report generated.')
send_analysis_result_email(analysis)
return success
def process_impact_result(self, impact_result, analysis_id):
"""Extract impact analysis after running it via InaSAFE-Headless celery
:param self: Task instance
:type self: celery.task.Task
:param impact_result: A dictionary of output's layer key and Uri with
status and message.
:type impact_result: dict
:param analysis_id: analysis id of the object
:type analysis_id: int
:return: True if success
:rtype: bool
"""
# Track the current task_id
analysis = Analysis.objects.get(id=analysis_id)
analysis.task_id = self.request.id
analysis.save()
success = False
report_success = False
impact_url = None
impact_path = None
if impact_result['status'] == RESULT_SUCCESS:
impact_url = (
impact_result['output'].get('impact_analysis')
or impact_result['output'].get('hazard_aggregation_summary'))
analysis_summary_url = (
impact_result['output'].get('analysis_summary'))
# generate report when analysis has ran successfully
async = generate_report.delay(impact_url)
with allow_join_result():
report_metadata = async .get().get('output', {})
for product_key, products in report_metadata.iteritems():
for report_key, report_url in products.iteritems():
report_url = download_file(report_url, direct_access=True)
report_metadata[product_key][report_key] = report_url
# decide if we are using direct access or not
impact_url = get_impact_path(impact_url)
# download impact layer path
impact_path = download_file(impact_url, direct_access=True)
dir_name = os.path.dirname(impact_path)
is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip'
if is_zipfile:
# Extract the layer first
with ZipFile(impact_path) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if ext in cov_exts + vec_exts:
# process this in the for loop to make sure it
# works only when we found the layer
success = process_impact_layer(analysis, dir_name,
basename, name)
report_success = process_impact_report(
analysis, report_metadata)
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except BaseException:
pass
else:
# It means it is accessing an shp or tif directly
analysis_summary_filename = os.path.basename(analysis_summary_url)
impact_filename = os.path.basename(impact_path)
impact_basename, ext = os.path.splitext(impact_filename)
success = process_impact_layer(analysis, dir_name, impact_basename,
impact_filename,
analysis_summary_filename)
report_success = process_impact_report(analysis, report_metadata)
# cleanup
for name in os.listdir(dir_name):
filepath = os.path.join(dir_name, name)
is_file = os.path.isfile(filepath)
should_delete = name.split('.')[0] == impact_basename
if is_file and should_delete:
try:
os.remove(filepath)
except BaseException:
pass
# cleanup
try:
os.remove(impact_path)
except BaseException:
pass
if not success:
LOGGER.info('No impact layer found in {0}'.format(impact_url))
if not report_success:
LOGGER.info('No impact report generated.')
return success
def process_impact_result(self, impact_url, analysis_id):
"""Extract impact analysis after running it via InaSAFE-Headless celery
:param self: Task instance
:type self: celery.task.Task
:param impact_url: impact url returned from analysis
:type impact_url: str
:param analysis_id: analysis id of the object
:type analysis_id: int
:return: True if success
:rtype: bool
"""
# Track the current task_id
analysis = Analysis.objects.get(id=analysis_id)
analysis.task_id = self.request.id
analysis.save()
# decide if we are using direct access or not
impact_url = get_impact_path(impact_url)
# download impact zip
impact_path = download_file(impact_url)
dir_name = os.path.dirname(impact_path)
success = False
with ZipFile(impact_path) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if ext in ['.shp', '.tif']:
# process this in the for loop to make sure it works only
# when we found the layer
saved_layer = file_upload(os.path.join(dir_name, name),
overwrite=True)
saved_layer.set_default_permissions()
if analysis.user_title:
layer_name = analysis.user_title
else:
layer_name = analysis.get_default_impact_title()
saved_layer.title = layer_name
saved_layer.save()
current_impact = None
if analysis.impact_layer:
current_impact = analysis.impact_layer
analysis.impact_layer = saved_layer
# check map report and table
report_map_path = os.path.join(dir_name, '%s.pdf' % basename)
if os.path.exists(report_map_path):
analysis.assign_report_map(report_map_path)
report_table_path = os.path.join(dir_name,
'%s_table.pdf' % basename)
if os.path.exists(report_table_path):
analysis.assign_report_table(report_table_path)
analysis.task_id = process_impact_result.request.id
analysis.task_state = 'SUCCESS'
analysis.save()
if current_impact:
current_impact.delete()
success = True
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except:
pass
# cleanup
try:
os.remove(impact_path)
except:
pass
if not success:
LOGGER.info('No impact layer found in %s' % impact_url)
return success
def process_impact_result(self, impact_url, analysis_id):
"""Extract impact analysis after running it via InaSAFE-Headless celery
:param self: Task instance
:type self: celery.task.Task
:param impact_url: impact url returned from analysis
:type impact_url: str
:param analysis_id: analysis id of the object
:type analysis_id: int
:return: True if success
:rtype: bool
"""
# Track the current task_id
analysis = Analysis.objects.get(id=analysis_id)
analysis.task_id = self.request.id
analysis.save()
# decide if we are using direct access or not
impact_url = get_impact_path(impact_url)
# download impact layer path
impact_path = download_file(impact_url, direct_access=True)
dir_name = os.path.dirname(impact_path)
success = False
is_zipfile = os.path.splitext(impact_path)[1].lower() == '.zip'
if is_zipfile:
# Extract the layer first
with ZipFile(impact_path) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if ext in ['.shp', '.tif']:
# process this in the for loop to make sure it works only
# when we found the layer
success = process_impact_layer(
analysis, basename, dir_name, name)
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except BaseException:
pass
else:
# It means it is accessing an shp or tif directly
filename = os.path.basename(impact_path)
basename, ext = os.path.splitext(filename)
success = process_impact_layer(analysis, basename, dir_name, filename)
# cleanup
for name in os.listdir(dir_name):
filepath = os.path.join(dir_name, name)
is_file = os.path.isfile(filepath)
should_delete = name.split('.')[0] == basename
if is_file and should_delete:
try:
os.remove(filepath)
except BaseException:
pass
# cleanup
try:
os.remove(impact_path)
except BaseException:
pass
if not success:
LOGGER.info('No impact layer found in %s' % impact_url)
return success
