def export_loss_map_csv(key, output, target): """ Export `output` to `target` in CSV format """ dest = _get_result_export_dest(target, output, file_ext=key[1]) data = [] for row in models.order_by_location( output.loss_map.lossmapdata_set.all().order_by('asset_ref')): data.append(scientific.LossMapPerAsset(row.asset_ref, row.value)) header = [data[0]._fields] writers.save_csv(dest, header + data, fmt='%10.6E') return dest
def export_loss_map_csv(key, output, target): """ Export `output` to `target` in CSV format """ dest = _get_result_export_dest(target, output, file_ext=key[1]) data = [] for row in models.order_by_location( output.loss_map.lossmapdata_set.all().order_by('asset_ref')): data.append(LossMapPerAsset(row.asset_ref, row.value)) header = [data[0]._fields] writers.save_csv(dest, header + data, fmt='%10.6E') return dest
def export_loss_map(key, output, target): """ General loss map export code. """ args = _export_common(output, output.loss_map.loss_type) dest = _get_result_export_dest(target, output, file_ext=key[1]) args.update( dict(poe=output.loss_map.poe, loss_category=output.oq_job.exposure_model.category)) writercls = (risk_writers.LossMapXMLWriter if key[1] == 'xml' else risk_writers.LossMapGeoJSONWriter) writercls(dest, **args).serialize( models.order_by_location( output.loss_map.lossmapdata_set.all().order_by('asset_ref'))) return dest
def export_loss_map(key, output, target): """ General loss map export code. """ args = _export_common(output, output.loss_map.loss_type) dest = _get_result_export_dest(target, output, file_ext=key[1]) args.update(dict( poe=output.loss_map.poe, loss_category=output.oq_job.exposure_model.category)) writercls = (risk_writers.LossMapXMLWriter if key[1] == 'xml' else risk_writers.LossMapGeoJSONWriter) writercls(dest, **args).serialize( models.order_by_location( output.loss_map.lossmapdata_set.all().order_by('asset_ref'))) return dest
def _export_loss_map(output, target, writer_class, file_ext): """ General loss map export code. """ risk_calculation = output.oq_job.risk_calculation args = _export_common(output, output.loss_map.loss_type) dest = _get_result_export_dest(target, output, file_ext=file_ext) args.update( dict(poe=output.loss_map.poe, loss_category=risk_calculation.exposure_model.category)) writer = writer_class(dest, **args) writer.serialize( models.order_by_location( output.loss_map.lossmapdata_set.all().order_by('asset_ref'))) return dest
def _export_loss_map(output, target, writer_class, file_ext): """ General loss map export code. """ risk_calculation = output.oq_job.risk_calculation args = _export_common(output, output.loss_map.loss_type) dest = _get_result_export_dest(target, output, file_ext=file_ext) args.update(dict( poe=output.loss_map.poe, loss_category=risk_calculation.exposure_model.category)) writer = writer_class(dest, **args) writer.serialize( models.order_by_location( output.loss_map.lossmapdata_set.all().order_by('asset_ref'))) return dest
def do_aggregate_post_proc(self): """ Grab hazard data for all realizations and sites from the database and compute mean and/or quantile aggregates (depending on which options are enabled in the calculation). Post-processing results will be stored directly into the database. """ num_rlzs = models.LtRealization.objects.filter( lt_model__hazard_calculation=self.hc).count() num_site_blocks_per_incr = int(CURVE_CACHE_SIZE) / int(num_rlzs) if num_site_blocks_per_incr == 0: # This means we have `num_rlzs` >= `CURVE_CACHE_SIZE`. # The minimum number of sites should be 1. num_site_blocks_per_incr = 1 slice_incr = num_site_blocks_per_incr * num_rlzs # unit: num records if self.hc.mean_hazard_curves: # create a new `HazardCurve` 'container' record for mean # curves (virtual container for multiple imts) models.HazardCurve.objects.create( output=models.Output.objects.create_output( self.job, "mean-curves-multi-imt", "hazard_curve_multi"), statistics="mean", imt=None, investigation_time=self.hc.investigation_time) if self.hc.quantile_hazard_curves: for quantile in self.hc.quantile_hazard_curves: # create a new `HazardCurve` 'container' record for quantile # curves (virtual container for multiple imts) models.HazardCurve.objects.create( output=models.Output.objects.create_output( self.job, 'quantile(%s)-curves' % quantile, "hazard_curve_multi"), statistics="quantile", imt=None, quantile=quantile, investigation_time=self.hc.investigation_time) for imt, imls in self.hc.intensity_measure_types_and_levels.items(): im_type, sa_period, sa_damping = from_string(imt) # prepare `output` and `hazard_curve` containers in the DB: container_ids = dict() if self.hc.mean_hazard_curves: mean_output = models.Output.objects.create_output( job=self.job, display_name='Mean Hazard Curves %s' % imt, output_type='hazard_curve' ) mean_hc = models.HazardCurve.objects.create( output=mean_output, investigation_time=self.hc.investigation_time, imt=im_type, imls=imls, sa_period=sa_period, sa_damping=sa_damping, statistics='mean' ) container_ids['mean'] = mean_hc.id if self.hc.quantile_hazard_curves: for quantile in self.hc.quantile_hazard_curves: q_output = models.Output.objects.create_output( job=self.job, display_name=( '%s quantile Hazard Curves %s' % (quantile, imt) ), output_type='hazard_curve' ) q_hc = models.HazardCurve.objects.create( output=q_output, investigation_time=self.hc.investigation_time, imt=im_type, imls=imls, sa_period=sa_period, sa_damping=sa_damping, statistics='quantile', quantile=quantile ) container_ids['q%s' % quantile] = q_hc.id all_curves_for_imt = models.order_by_location( models.HazardCurveData.objects.all_curves_for_imt( self.job.id, im_type, sa_period, sa_damping)) with transaction.commit_on_success(using='job_init'): inserter = writer.CacheInserter( models.HazardCurveData, CURVE_CACHE_SIZE) for chunk in models.queryset_iter(all_curves_for_imt, slice_incr): # slice each chunk by `num_rlzs` into `site_chunk` # and compute the aggregate for site_chunk in block_splitter(chunk, num_rlzs): site = site_chunk[0].location curves_poes = [x.poes for x in site_chunk] curves_weights = [x.weight for x in site_chunk] # do means and quantiles # quantiles first: if self.hc.quantile_hazard_curves: for quantile in self.hc.quantile_hazard_curves: if self.hc.number_of_logic_tree_samples == 0: # explicitly weighted quantiles q_curve = weighted_quantile_curve( curves_poes, curves_weights, quantile ) else: # implicitly weighted quantiles q_curve = quantile_curve( curves_poes, quantile ) inserter.add( models.HazardCurveData( hazard_curve_id=( container_ids['q%s' % quantile]), poes=q_curve.tolist(), location=site.wkt) ) # then means if self.hc.mean_hazard_curves: m_curve = mean_curve( curves_poes, weights=curves_weights ) inserter.add( models.HazardCurveData( hazard_curve_id=container_ids['mean'], poes=m_curve.tolist(), location=site.wkt) ) inserter.flush()
def do_aggregate_post_proc(self): """ Grab hazard data for all realizations and sites from the database and compute mean and/or quantile aggregates (depending on which options are enabled in the calculation). Post-processing results will be stored directly into the database. """ num_rlzs = models.LtRealization.objects.filter( hazard_calculation=self.hc).count() num_site_blocks_per_incr = int(CURVE_CACHE_SIZE) / int(num_rlzs) if num_site_blocks_per_incr == 0: # This means we have `num_rlzs` >= `CURVE_CACHE_SIZE`. # The minimum number of sites should be 1. num_site_blocks_per_incr = 1 slice_incr = num_site_blocks_per_incr * num_rlzs # unit: num records if self.hc.mean_hazard_curves: # create a new `HazardCurve` 'container' record for mean # curves (virtual container for multiple imts) models.HazardCurve.objects.create( output=models.Output.objects.create_output( self.job, "mean-curves-multi-imt", "hazard_curve_multi"), statistics="mean", imt=None, investigation_time=self.hc.investigation_time) if self.hc.quantile_hazard_curves: for quantile in self.hc.quantile_hazard_curves: # create a new `HazardCurve` 'container' record for quantile # curves (virtual container for multiple imts) models.HazardCurve.objects.create( output=models.Output.objects.create_output( self.job, 'quantile(%s)-curves' % quantile, "hazard_curve_multi"), statistics="quantile", imt=None, quantile=quantile, investigation_time=self.hc.investigation_time) for imt, imls in self.hc.intensity_measure_types_and_levels.items(): im_type, sa_period, sa_damping = models.parse_imt(imt) # prepare `output` and `hazard_curve` containers in the DB: container_ids = dict() if self.hc.mean_hazard_curves: mean_output = models.Output.objects.create_output( job=self.job, display_name='mean-curves-%s' % imt, output_type='hazard_curve') mean_hc = models.HazardCurve.objects.create( output=mean_output, investigation_time=self.hc.investigation_time, imt=im_type, imls=imls, sa_period=sa_period, sa_damping=sa_damping, statistics='mean') container_ids['mean'] = mean_hc.id if self.hc.quantile_hazard_curves: for quantile in self.hc.quantile_hazard_curves: q_output = models.Output.objects.create_output( job=self.job, display_name=('quantile(%s)-curves-%s' % (quantile, imt)), output_type='hazard_curve') q_hc = models.HazardCurve.objects.create( output=q_output, investigation_time=self.hc.investigation_time, imt=im_type, imls=imls, sa_period=sa_period, sa_damping=sa_damping, statistics='quantile', quantile=quantile) container_ids['q%s' % quantile] = q_hc.id all_curves_for_imt = models.order_by_location( models.HazardCurveData.objects.all_curves_for_imt( self.job.id, im_type, sa_period, sa_damping)) with transaction.commit_on_success(using='reslt_writer'): inserter = writer.CacheInserter(models.HazardCurveData, CURVE_CACHE_SIZE) for chunk in models.queryset_iter(all_curves_for_imt, slice_incr): # slice each chunk by `num_rlzs` into `site_chunk` # and compute the aggregate for site_chunk in block_splitter(chunk, num_rlzs): site = site_chunk[0].location curves_poes = [x.poes for x in site_chunk] curves_weights = [x.weight for x in site_chunk] # do means and quantiles # quantiles first: if self.hc.quantile_hazard_curves: for quantile in self.hc.quantile_hazard_curves: if self.hc.number_of_logic_tree_samples == 0: # explicitly weighted quantiles q_curve = weighted_quantile_curve( curves_poes, curves_weights, quantile) else: # implicitly weighted quantiles q_curve = quantile_curve( curves_poes, quantile) inserter.add( models.HazardCurveData( hazard_curve_id=( container_ids['q%s' % quantile]), poes=q_curve.tolist(), location=site.wkt)) # then means if self.hc.mean_hazard_curves: m_curve = mean_curve(curves_poes, weights=curves_weights) inserter.add( models.HazardCurveData( hazard_curve_id=container_ids['mean'], poes=m_curve.tolist(), location=site.wkt)) inserter.flush()