def _filter_agg(assetcol, losses, selected, stats=''): # losses is an array of shape (A, ..., R) with A=#assets, R=#realizations aids_by_tag = assetcol.get_aids_by_tag() idxs = set(range(len(assetcol))) tagnames = [] for tag in selected: tagname, tagvalue = tag.split('=', 1) if tagvalue == '*': tagnames.append(tagname) else: idxs &= aids_by_tag[tag] if len(tagnames) > 1: raise ValueError('Too many * as tag values in %s' % tagnames) elif not tagnames: # return an array of shape (..., R) return ArrayWrapper( _agg(losses, idxs), dict(selected=encode(selected), stats=stats)) else: # return an array of shape (T, ..., R) [tagname] = tagnames _tags = list(assetcol.tagcol.gen_tags(tagname)) all_idxs = [idxs & aids_by_tag[t] for t in _tags] # NB: using a generator expression for all_idxs caused issues (?) data, tags = [], [] for idxs, tag in zip(all_idxs, _tags): agglosses = _agg(losses, idxs) if len(agglosses): data.append(agglosses) tags.append(tag) return ArrayWrapper( numpy.array(data), dict(selected=encode(selected), tags=encode(tags), stats=stats))
def build_datasets(self, builder): oq = self.oqparam stats = oq.hazard_stats().items() S = len(stats) P = len(builder.return_periods) C = len(oq.conditional_loss_poes) loss_types = ' '.join(self.riskmodel.loss_types) shp = self.get_shape(P, self.R, self.L) # shape P, R, L, T... self.datastore.create_dset('agg_curves-rlzs', F32, shp) self.datastore.set_attrs( 'agg_curves-rlzs', return_periods=builder.return_periods, loss_types=loss_types) if oq.conditional_loss_poes: shp = self.get_shape(C, self.R, self.L) # shape C, R, L, T... self.datastore.create_dset('agg_maps-rlzs', F32, shp) if self.R > 1: shp = self.get_shape(P, S, self.L) # shape P, S, L, T... self.datastore.create_dset('agg_curves-stats', F32, shp) self.datastore.set_attrs( 'agg_curves-stats', return_periods=builder.return_periods, stats=[encode(name) for (name, func) in stats], loss_types=loss_types) if oq.conditional_loss_poes: shp = self.get_shape(C, S, self.L) # shape C, S, L, T... self.datastore.create_dset('agg_maps-stats', F32, shp) self.datastore.set_attrs( 'agg_maps-stats', stats=[encode(name) for (name, func) in stats], loss_types=loss_types)
def build_datasets(self, builder): oq = self.oqparam R = len(builder.weights) stats = self.param['stats'] A = self.A S = len(stats) P = len(builder.return_periods) C = len(oq.conditional_loss_poes) L = self.L self.loss_maps_dt = (F32, (C, L)) if oq.individual_curves or R == 1: self.datastore.create_dset('curves-rlzs', F32, (A, R, P, L)) self.datastore.set_attrs( 'curves-rlzs', return_periods=builder.return_periods) if oq.conditional_loss_poes: self.datastore.create_dset( 'loss_maps-rlzs', self.loss_maps_dt, (A, R), fillvalue=None) if R > 1: self.datastore.create_dset('curves-stats', F32, (A, S, P, L)) self.datastore.set_attrs( 'curves-stats', return_periods=builder.return_periods, stats=[encode(name) for (name, func) in stats]) if oq.conditional_loss_poes: self.datastore.create_dset( 'loss_maps-stats', self.loss_maps_dt, (A, S), fillvalue=None) self.datastore.set_attrs( 'loss_maps-stats', stats=[encode(name) for (name, func) in stats])
def post_execute(self, results): """ Save all the results of the disaggregation. NB: the number of results to save is #sites * #rlzs * #disagg_poes * #IMTs. :param results: a dictionary (sid, rlzi, poe, imt) -> trti -> disagg matrix """ T = len(self.trts) # build a dictionary (sid, rlzi, poe, imt) -> 6D matrix results = {k: _to_matrix(v, T) for k, v in results.items()} # get the number of outputs shp = self.get_NRPM() logging.info('Extracting and saving the PMFs for %d outputs ' '(N=%s, R=%d, P=%d, M=%d)', numpy.prod(shp), *shp) self.save_disagg_result('disagg', results) hstats = self.oqparam.hazard_stats() if len(self.rlzs_assoc.realizations) > 1 and hstats: with self.monitor('computing and saving stats', measuremem=True): res = self.build_stats(results, hstats.items()) self.save_disagg_result('disagg-stats', res) self.datastore.set_attrs( 'disagg', trts=encode(self.trts), num_ruptures=self.num_ruptures)
def store_source_info(self, pmap_by_grp_id): # store the information about received data received = self.taskman.received if received: tname = self.taskman.name self.datastore.save('job_info', { tname + '_max_received_per_task': max(received), tname + '_tot_received': sum(received), tname + '_num_tasks': len(received)}) # then save the calculation times per each source calc_times = getattr(pmap_by_grp_id, 'calc_times', []) if calc_times: sources = self.csm.get_sources() info_dict = {(rec['src_group_id'], rec['source_id']): rec for rec in self.source_info} for src_idx, dt in calc_times: src = sources[src_idx] info = info_dict[src.src_group_id, encode(src.source_id)] info['calc_time'] += dt rows = sorted( info_dict.values(), key=operator.itemgetter(7), reverse=True) array = numpy.zeros(len(rows), source.source_info_dt) for i, row in enumerate(rows): for name in array.dtype.names: array[i][name] = row[name] self.source_info = array self.datastore.hdf5.flush()
def maybe_encode(value): """ If value is a sequence of strings, encode it """ if isinstance(value, (list, tuple)): if value and isinstance(value[0], str): return encode(value) return value
def get_gmfs_from_txt(oqparam, fname): """ :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param fname: the full path of the CSV file :returns: a composite array of shape (N, R) read from a CSV file with format `etag indices [gmv1 ... gmvN] * num_imts` """ with open(fname) as csvfile: firstline = next(csvfile) try: coords = valid.coordinates(firstline) except: raise InvalidFile( 'The first line of %s is expected to contain comma separated' 'ordered coordinates, got %s instead' % (fname, firstline)) sitecol = sitecol_from_coords(oqparam, coords) if not oqparam.imtls: oqparam.set_risk_imtls(get_risk_models(oqparam)) imts = list(oqparam.imtls) imt_dt = numpy.dtype([(imt, F32) for imt in imts]) num_gmfs = oqparam.number_of_ground_motion_fields gmf_by_imt = numpy.zeros((num_gmfs, len(sitecol)), imt_dt) etags = [] for lineno, line in enumerate(csvfile, 2): row = line.split(',') try: indices = list(map(valid.positiveint, row[1].split())) except: raise InvalidFile( 'The second column in %s is expected to contain integer ' 'indices, got %s' % (fname, row[1])) r_sites = ( sitecol if not indices else site.FilteredSiteCollection(indices, sitecol)) for i in range(len(imts)): try: array = numpy.array(valid.positivefloats(row[i + 2])) # NB: i + 2 because the first 2 fields are etag and indices except: raise InvalidFile( 'The column #%d in %s is expected to contain positive ' 'floats, got %s instead' % (i + 3, fname, row[i + 2])) gmf_by_imt[imts[i]][lineno - 2] = r_sites.expand(array, 0) etags.append(row[0]) if lineno < num_gmfs + 1: raise InvalidFile('%s contains %d rows, expected %d' % ( fname, lineno, num_gmfs + 1)) if etags != sorted(etags): raise InvalidFile('The etags in %s are not ordered: %s' % (fname, etags)) return sitecol, numpy.array([encode(e) for e in etags]), gmf_by_imt.T
def __toh5__(self): # NB: the loss types do not contain spaces, so we can store them # together as a single space-separated string op = decode(self.occupancy_periods) attrs = {'time_event': self.time_event or 'None', 'occupancy_periods': op, 'tot_sites': self.tot_sites, 'fields': ' '.join(self.fields), 'tagnames': encode(self.tagnames), 'nbytes': self.array.nbytes} return dict(array=self.array, tagcol=self.tagcol), attrs
def post_execute(self, num_events): """ Save risk data and possibly execute the EbrPostCalculator """ # gmv[:-2] are the total gmv per each IMT gmv = sum(gm[:-2].sum() for gm in self.gmdata.values()) if not gmv: raise RuntimeError('No GMFs were generated, perhaps they were ' 'all below the minimum_intensity threshold') if 'agg_loss_table' not in self.datastore: logging.warning( 'No losses were generated: most likely there is an error in y' 'our input files or the GMFs were below the minimum intensity') else: self.datastore.set_nbytes('agg_loss_table') E = sum(num_events.values()) agglt = self.datastore['agg_loss_table'] agglt.attrs['nonzero_fraction'] = len(agglt) / E # build aggregate loss curves self.before_export() # set 'realizations' oq = self.oqparam b = get_loss_builder(self.datastore) alt = self.datastore['agg_loss_table'] stats = oq.risk_stats() array, array_stats = b.build(alt, stats) self.datastore['agg_curves-rlzs'] = array units = self.assetcol.units(loss_types=array.dtype.names) self.datastore.set_attrs('agg_curves-rlzs', return_periods=b.return_periods, units=units) if array_stats is not None: self.datastore['agg_curves-stats'] = array_stats self.datastore.set_attrs( 'agg_curves-stats', return_periods=b.return_periods, stats=[encode(name) for (name, func) in stats], units=units) if 'all_loss_ratios' in self.datastore: self.datastore.save_vlen('all_loss_ratios/indices', [ numpy.array(self.indices[aid], riskinput.indices_dt) for aid in range(self.A) ]) self.datastore.set_attrs('all_loss_ratios', loss_types=' '.join( self.riskmodel.loss_types)) dset = self.datastore['all_loss_ratios/data'] nbytes = dset.size * dset.dtype.itemsize self.datastore.set_attrs('all_loss_ratios/data', nbytes=nbytes, bytes_per_asset=nbytes / self.A) EbrPostCalculator(self).run(close=False)
def extract_task_info(dstore, what): """ Extracts the task distribution. Use it as /extract/task_info?kind=classical """ dic = group_array(dstore['task_info'][()], 'taskname') if 'kind' in what: name = parse(what)['kind'][0] yield name, dic[encode(name)] return for name in dic: yield decode(name), dic[name]
def get_gmfs_from_txt(oqparam, fname): """ :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param fname: the full path of the CSV file :returns: a composite array of shape (N, R) read from a CSV file with format `etag indices [gmv1 ... gmvN] * num_imts` """ with open(fname) as csvfile: firstline = next(csvfile) try: coords = valid.coordinates(firstline) except: raise InvalidFile( 'The first line of %s is expected to contain comma separated' 'ordered coordinates, got %s instead' % (fname, firstline)) lons, lats, depths = zip(*coords) sitecol = site.SiteCollection.from_points(lons, lats, depths, oqparam) if not oqparam.imtls: oqparam.set_risk_imtls(get_risk_models(oqparam)) imts = list(oqparam.imtls) imt_dt = numpy.dtype([(imt, F32) for imt in imts]) num_gmfs = oqparam.number_of_ground_motion_fields gmf_by_imt = numpy.zeros((num_gmfs, len(sitecol)), imt_dt) eids = [] for lineno, line in enumerate(csvfile, 2): row = line.split(',') try: indices = list(map(valid.positiveint, row[1].split())) except: raise InvalidFile( 'The second column in %s is expected to contain integer ' 'indices, got %s' % (fname, row[1])) r_sites = (sitecol if not indices else site.FilteredSiteCollection( indices, sitecol)) for i in range(len(imts)): try: array = numpy.array(valid.positivefloats(row[i + 2])) # NB: i + 2 because the first 2 fields are etag and indices except: raise InvalidFile( 'The column #%d in %s is expected to contain positive ' 'floats, got %s instead' % (i + 3, fname, row[i + 2])) gmf_by_imt[imts[i]][lineno - 2][r_sites.sids] = array eids.append(row[0]) if lineno < num_gmfs + 1: raise InvalidFile('%s contains %d rows, expected %d' % (fname, lineno, num_gmfs + 1)) if eids != sorted(eids): raise InvalidFile('The eids in %s are not ordered: %s' % (fname, eids)) return sitecol, numpy.array([encode(e) for e in eids]), gmf_by_imt.T
def _display(node, indent, expandattrs, expandvals, output): """Core function to display a Node object""" attrs = _displayattrs(node.attrib, expandattrs) if node.text is None or not expandvals: val = '' elif isinstance(node.text, str): val = ' %s' % repr(node.text.strip()) else: val = ' %s' % repr(node.text) # node.text can be a tuple output.write(encode(indent + striptag(node.tag) + attrs + val + '\n')) for sub_node in node: _display(sub_node, indent + ' ', expandattrs, expandvals, output)
def postproc(self): """ Build aggregate loss curves and run EbrPostCalculator """ dstore = self.datastore self.before_export() # set 'realizations' oq = self.oqparam eff_time = oq.investigation_time * oq.ses_per_logic_tree_path if eff_time < 2: logging.warn('eff_time=%s is too small to compute agg_curves', eff_time) return stats = oq.risk_stats() # store avg_losses-stats if oq.avg_losses: set_rlzs_stats(self.datastore, 'avg_losses') b = get_loss_builder(dstore) if 'ruptures' in dstore: logging.info('Building rup_loss_table') with self.monitor('building rup_loss_table', measuremem=True): dstore['rup_loss_table'] = rlt = build_rup_loss_table(dstore) ridx = [rlt[lt].argmax() for lt in oq.loss_dt().names] dstore.set_attrs('rup_loss_table', ridx=ridx) logging.info('Building aggregate loss curves') with self.monitor('building agg_curves', measuremem=True): array, array_stats = b.build(dstore['agg_loss_table'].value, stats) self.datastore['agg_curves-rlzs'] = array units = self.assetcol.units(loss_types=array.dtype.names) self.datastore.set_attrs('agg_curves-rlzs', return_periods=b.return_periods, units=units) if array_stats is not None: self.datastore['agg_curves-stats'] = array_stats self.datastore.set_attrs( 'agg_curves-stats', return_periods=b.return_periods, stats=[encode(name) for (name, func) in stats], units=units) if 'all_loss_ratios' in self.datastore: self.datastore.save_vlen('all_loss_ratios/indices', [ numpy.array(self.indices[aid], riskinput.indices_dt) for aid in range(self.A) ]) self.datastore.set_attrs('all_loss_ratios', loss_types=' '.join( self.riskmodel.loss_types)) dset = self.datastore['all_loss_ratios/data'] nbytes = dset.size * dset.dtype.itemsize self.datastore.set_attrs('all_loss_ratios/data', nbytes=nbytes, bytes_per_asset=nbytes / self.A) EbrPostCalculator(self).run(close=False)
def sanitize(value): """ Sanitize the value so that it can be stored as an HDF5 attribute """ if isinstance(value, bytes): return numpy.void(value) elif isinstance(value, (list, tuple)): if value and isinstance(value[0], str): return encode(value) elif isinstance(value, int) and value > sys.maxsize: return float(value) return value
def etags(self): """ An array of tags for the underlying seismic events """ tags = [] for (eid, ses, occ, sampleid) in self.events: tag = 'grp=%02d~ses=%04d~src=%s~rup=%d-%02d' % ( self.grp_id, ses, self.source_id, self.serial, occ) if sampleid > 0: tag += '~sample=%d' % sampleid tags.append(encode(tag)) return numpy.array(tags)
def units(self, loss_types): """ :param: a list of loss types :returns: an array of units as byte strings, suitable for HDF5 """ units = self.cc.units lst = [] for lt in loss_types: if lt.endswith('_ins'): lt = lt[:-4] lst.append(encode(units[lt])) return numpy.array(lst)
def build_datasets(self, builder): oq = self.oqparam R = len(builder.weights) stats = self.param['stats'] A = self.A S = len(stats) P = len(builder.return_periods) C = len(oq.conditional_loss_poes) L = self.L self.loss_maps_dt = (F32, (C, L)) if oq.individual_curves or R == 1: self.datastore.create_dset('curves-rlzs', F32, (A, R, P, L)) shape_descr = ['assets', 'rlzs', 'return_periods', 'loss_types'] self.datastore.set_attrs('curves-rlzs', shape_descr=shape_descr, assets=self.assetcol['id'], return_periods=builder.return_periods, rlzs=numpy.arange(R), loss_types=oq.loss_names) if oq.conditional_loss_poes: self.datastore.create_dset('loss_maps-rlzs', self.loss_maps_dt, (A, R), fillvalue=None) if R > 1: self.datastore.create_dset('curves-stats', F32, (A, S, P, L)) shape_descr = ['assets', 'stats', 'return_periods', 'loss_types'] self.datastore.set_attrs( 'curves-stats', shape_descr=shape_descr, assets=self.assetcol['id'], stats=[encode(name) for (name, func) in stats], return_periods=builder.return_periods, loss_types=oq.loss_names) if oq.conditional_loss_poes: self.datastore.create_dset('loss_maps-stats', self.loss_maps_dt, (A, S), fillvalue=None) self.datastore.set_attrs( 'loss_maps-stats', stats=[encode(name) for (name, func) in stats])
def saving_sources_by_task(iterargs, dstore): """ Yield the iterargs again by populating 'source_data' """ source_ids = [] data = [] for i, args in enumerate(iterargs, 1): source_ids.append(get_src_ids(args[0])) for src in args[0]: # collect source data data.append((i, src.nsites, src.num_ruptures, src.weight)) yield args dstore['task_sources'] = encode(source_ids) dstore.extend('source_data', numpy.array(data, source_data_dt))
def saving_sources_by_task(iterargs, dstore): """ Yield the iterargs again by populating 'task_info/source_ids' """ source_ids = [] data = [] for i, args in enumerate(iterargs, 1): source_ids.append(' '.join(src.source_id for src in args[0])) for src in args[0]: # collect source data data.append((i, src.nsites, src.weight)) yield args dstore['task_sources'] = numpy.array([encode(s) for s in source_ids]) dstore.extend('task_info/source_data', numpy.array(data, source_data_dt))
def __toh5__(self): # NB: the loss types do not contain spaces, so we can store them # together as a single space-separated string op = decode(self.occupancy_periods) attrs = {'time_event': self.time_event or 'None', 'occupancy_periods': op, 'loss_types': ' '.join(self.loss_types), 'tot_sites': self.tot_sites, 'fields': ' '.join(self.fields), 'tagnames': encode(self.tagnames), 'nbytes': self.array.nbytes} return dict( array=self.array, tagcol=self.tagcol), attrs
def post_execute(self, result): """ Compute stats for the aggregated distributions and save the results on the datastore. """ loss_dt = self.oqparam.loss_dt() LI = len(loss_dt.names) dtlist = [('eid', U64), ('rlzi', U16), ('loss', (F32, LI))] I = self.oqparam.insured_losses + 1 R = self.R with self.monitor('saving outputs', autoflush=True): A = len(self.assetcol) # agg losses res = result['agg'] E, LI = res.shape L = LI // I mean, std = scientific.mean_std(res) # shape LI agglosses = numpy.zeros(LI, stat_dt) agglosses['mean'] = F32(mean) agglosses['stddev'] = F32(std) # losses by asset losses_by_asset = numpy.zeros((A, R, LI), stat_dt) for (l, r, aid, stat) in result['avg']: for i in range(I): losses_by_asset[aid, r, l + L * i] = stat[i] self.datastore['losses_by_asset'] = losses_by_asset self.datastore['agglosses'] = agglosses # losses by event num_gmfs = self.oqparam.number_of_ground_motion_fields lbe = numpy.fromiter( ((ei, ei // num_gmfs, res[ei]) for ei in range(E)), dtlist) self.datastore['losses_by_event'] = lbe loss_types = ' '.join(self.oqparam.loss_dt().names) self.datastore.set_attrs('losses_by_event', loss_types=loss_types) # all losses if self.oqparam.asset_loss_table: array = numpy.zeros((A, E), loss_dt) for (l, r), losses_by_aid in result['all_losses'].items(): slc = self.event_slice(r) for aid in losses_by_aid: lba = losses_by_aid[aid] # (E, I) for i in range(I): lt = loss_dt.names[l + L * i] array[lt][aid, slc] = lba[:, i] self.datastore['asset_loss_table'] = array tags = [encode(tag) for tag in self.assetcol.tagcol] self.datastore.set_attrs('asset_loss_table', tags=tags)
def post_execute(self, result): """ Compute stats for the aggregated distributions and save the results on the datastore. """ loss_dt = self.oqparam.loss_dt() L = len(loss_dt.names) dtlist = [('event_id', U32), ('rlzi', U16), ('loss', (F32, (L, )))] R = self.R with self.monitor('saving outputs'): A = len(self.assetcol) # agg losses res = result['agg'] E, L = res.shape agglosses = numpy.zeros((R, L), stat_dt) for r in range(R): mean, std = scientific.mean_std(res[self.event_slice(r)]) agglosses[r]['mean'] = F32(mean) agglosses[r]['stddev'] = F32(std) # losses by asset losses_by_asset = numpy.zeros((A, R, L), stat_dt) for (l, r, aid, stat) in result['avg']: losses_by_asset[aid, r, l] = stat self.datastore['losses_by_asset'] = losses_by_asset self.datastore['agglosses'] = agglosses # losses by event lbe = numpy.zeros(E, dtlist) lbe['event_id'] = range(E) lbe['rlzi'] = (lbe['event_id'] // self.oqparam.number_of_ground_motion_fields) lbe['loss'] = res self.datastore['losses_by_event'] = lbe loss_types = self.oqparam.loss_dt().names self.datastore.set_attrs('losses_by_event', loss_types=loss_types) # all losses if self.oqparam.asset_loss_table: array = numpy.zeros((A, E), loss_dt) for (l, r), losses_by_aid in result['all_losses'].items(): slc = self.event_slice(r) for aid in losses_by_aid: lba = losses_by_aid[aid] # E lt = loss_dt.names[l] array[lt][aid, slc] = lba self.datastore['asset_loss_table'] = array tags = [encode(tag) for tag in self.assetcol.tagcol] self.datastore.set_attrs('asset_loss_table', tags=tags)
def view_job_info(token, dstore): """ Determine the amount of data transferred from the controller node to the workers and back in a classical calculation. """ data = [] task_info = dstore['task_info'][()] task_sent = ast.literal_eval(decode(dstore['task_sent'][()])) for task, dic in task_sent.items(): sent = sorted(dic.items(), key=operator.itemgetter(1), reverse=True) sent = ['%s=%s' % (k, humansize(v)) for k, v in sent[:3]] recv = get_array(task_info, taskname=encode(task))['received'].sum() data.append((task, ' '.join(sent), humansize(recv))) return numpy.array(data, dt('task sent received'))
def set_shape_attrs(hdf5file, dsetname, kw): """ Set shape attributes on a dataset (and possibly other attributes) """ dset = hdf5file[dsetname] S = len(dset.shape) if len(kw) < S: raise ValueError('The dataset %s has %d dimensions but you passed %d' ' axis' % (dsetname, S, len(kw))) dset.attrs['shape_descr'] = encode(list(kw))[:S] for k, v in kw.items(): dset.attrs[k] = v for d, k in enumerate(dset.attrs['shape_descr']): dset.dims[d].label = k # set dimension label
def build_datasets(self, builder, aggregate_by, prefix): """ Create the datasets agg_curves-XXX, tot_curves-XXX, agg_losses-XXX, tot_losses-XXX. """ P = len(builder.return_periods) aggby = {'aggregate_by': aggregate_by} for tagname in aggregate_by: aggby[tagname] = encode(getattr(self.tagcol, tagname)[1:]) shp = self.get_shape(self.L, self.R, aggregate_by=aggregate_by) # shape L, R, T... self.datastore.create_dset(prefix + 'losses-rlzs', F32, shp) shp = self.get_shape(P, self.R, self.L, aggregate_by=aggregate_by) # shape P, R, L, T... self.datastore.create_dset(prefix + 'curves-rlzs', F32, shp)
def get_units(self, loss_types): """ :param: a list of loss types :returns: an array of units as byte strings, suitable for HDF5 """ lst = [] for lt in loss_types: if lt.endswith('_ins'): lt = lt[:-4] if lt == 'occupants': unit = 'people' else: unit = self.units[lt] lst.append(encode(unit)) return numpy.array(lst)
def __toh5__(self): # NB: the loss types do not contain spaces, so we can store them # together as a single space-separated string op = decode(self.occupancy_periods) attrs = {'time_event': self.time_event or 'None', 'occupancy_periods': op, 'loss_types': ' '.join(self.loss_types), 'deduc': ' '.join(self.deduc), 'i_lim': ' '.join(self.i_lim), 'retro': ' '.join(self.retro), 'tot_sites': self.tot_sites, 'tagnames': encode(self.tagnames), 'nbytes': self.array.nbytes} return dict( array=self.array, cost_calculator=self.cost_calculator, tagcol=self.tagcol, asset_refs=self.asset_refs), attrs
def get_assets(dstore): """ :param dstore: a datastore with keys 'assetcol' :returns: an array of records (id, tag1, ..., tagN, lon, lat) """ assetcol = dstore['assetcol'] tagnames = sorted(tn for tn in assetcol.tagnames if tn != 'id') tag = {t: getattr(assetcol.tagcol, t) for t in tagnames} dtlist = [('id', '<S100')] for tagname in tagnames: dtlist.append((tagname, '<S100')) dtlist.extend([('lon', F32), ('lat', F32)]) asset_data = [] for a in assetcol.array: tup = tuple(python3compat.encode(tag[t][a[t]]) for t in tagnames) asset_data.append((a['id'],) + tup + (a['lon'], a['lat'])) return numpy.array(asset_data, dtlist)
def post_execute(self, result): """ Compute stats for the aggregated distributions and save the results on the datastore. """ loss_dt = self.oqparam.loss_dt() LI = len(loss_dt.names) dtlist = [('eid', U64), ('loss', (F32, LI))] R = self.R with self.monitor('saving outputs', autoflush=True): A = len(self.assetcol) # agg losses res = result['agg'] E, L = res.shape agglosses = numpy.zeros((R, L), stat_dt) for r in range(R): mean, std = scientific.mean_std(res[self.event_slice(r)]) agglosses[r]['mean'] = F32(mean) agglosses[r]['stddev'] = F32(std) # losses by asset losses_by_asset = numpy.zeros((A, R, L), stat_dt) for (l, r, aid, stat) in result['avg']: losses_by_asset[aid, r, l] = stat self.datastore['losses_by_asset'] = losses_by_asset self.datastore['agglosses'] = agglosses # losses by event lbe = numpy.fromiter(((ei, res[ei]) for ei in range(E)), dtlist) self.datastore['losses_by_event'] = lbe loss_types = ' '.join(self.oqparam.loss_dt().names) self.datastore.set_attrs('losses_by_event', loss_types=loss_types) # all losses if self.oqparam.asset_loss_table: array = numpy.zeros((A, E), loss_dt) for (l, r), losses_by_aid in result['all_losses'].items(): slc = self.event_slice(r) for aid in losses_by_aid: lba = losses_by_aid[aid] # E lt = loss_dt.names[l] array[lt][aid, slc] = lba self.datastore['asset_loss_table'] = array tags = [encode(tag) for tag in self.assetcol.tagcol] self.datastore.set_attrs('asset_loss_table', tags=tags)
def postproc(self): """ Build aggregate loss curves in process """ dstore = self.datastore self.before_export() # set 'realizations' oq = self.oqparam stats = self.param['stats'] # store avg_losses-stats if oq.avg_losses: set_rlzs_stats(self.datastore, 'avg_losses') try: b = self.param['builder'] except KeyError: # don't build auxiliary tables return if dstore.parent: dstore.parent.open('r') # to read the ruptures if 'ruptures' in self.datastore and len(self.datastore['ruptures']): logging.info('Building loss tables') with self.monitor('building loss tables', measuremem=True): rlt, lbr = build_loss_tables(dstore) dstore['rup_loss_table'] = rlt dstore['losses_by_rlzi'] = lbr ridx = [rlt[:, lti].argmax() for lti in range(self.L)] dstore.set_attrs('rup_loss_table', ridx=ridx) logging.info('Building aggregate loss curves') with self.monitor('building agg_curves', measuremem=True): lbr = group_array(dstore['losses_by_event'][()], 'rlzi') dic = {r: arr['loss'] for r, arr in lbr.items()} array, arr_stats = b.build(dic, stats) loss_types = ' '.join(oq.loss_dt().names) units = self.datastore['cost_calculator'].get_units(loss_types.split()) if oq.individual_curves or self.R == 1: self.datastore['agg_curves-rlzs'] = array # shape (P, R, L) self.datastore.set_attrs('agg_curves-rlzs', return_periods=b.return_periods, loss_types=loss_types, units=units) if arr_stats is not None: self.datastore['agg_curves-stats'] = arr_stats # shape (P, S, L) self.datastore.set_attrs( 'agg_curves-stats', return_periods=b.return_periods, stats=[encode(name) for (name, func) in stats], loss_types=loss_types, units=units)
def post_execute(self, results): """ Save all the results of the disaggregation. NB: the number of results to save is #sites * #rlzs * #disagg_poes * #IMTs. :param results: a dictionary sid -> trti -> disagg matrix """ T = len(self.trts) # build a dictionary m, s -> 8D matrix of shape (T, ..., E, P) results = {ms: _trt_matrix(dic, T) for ms, dic in results.items()} # get the number of outputs shp = (self.N, len(self.poes_disagg), len(self.imts), self.Z) logging.info('Extracting and saving the PMFs for %d outputs ' '(N=%s, P=%d, M=%d, Z=%d)', numpy.prod(shp), *shp) self.save_disagg_results(results, trts=encode(self.trts))
def extract_disagg_layer(dstore, what): """ Extract a disaggregation layer containing all sites and outputs Example: http://127.0.0.1:8800/v1/calc/30/extract/disagg_layer? """ qdict = parse(what) oq = dstore['oqparam'] oq.maximum_distance = filters.MagDepDistance(oq.maximum_distance) if 'kind' in qdict: kinds = qdict['kind'] else: kinds = oq.disagg_outputs sitecol = dstore['sitecol'] poes_disagg = oq.poes_disagg or (None, ) edges, shapedic = disagg.get_edges_shapedic(oq, sitecol, dstore['source_mags']) dt = _disagg_output_dt(shapedic, kinds, oq.imtls, poes_disagg) out = numpy.zeros(len(sitecol), dt) realizations = numpy.array(dstore['full_lt'].get_realizations()) hmap4 = dstore['hmap4'][:] best_rlzs = dstore['best_rlzs'][:] arr = {kind: dstore['disagg/' + kind][:] for kind in kinds} for sid, lon, lat, rec in zip(sitecol.sids, sitecol.lons, sitecol.lats, out): rlzs = realizations[best_rlzs[sid]] rec['site_id'] = sid rec['lon'] = lon rec['lat'] = lat rec['lon_bins'] = edges[2][sid] rec['lat_bins'] = edges[3][sid] for m, imt in enumerate(oq.imtls): ws = numpy.array([rlz.weight[imt] for rlz in rlzs]) ws /= ws.sum() # normalize to 1 for p, poe in enumerate(poes_disagg): for kind in kinds: key = '%s-%s-%s' % (kind, imt, poe) rec[key] = arr[kind][sid, m, p] @ ws rec['iml-%s-%s' % (imt, poe)] = hmap4[sid, m, p] return ArrayWrapper( out, dict(mag=edges[0], dist=edges[1], eps=edges[-2], trt=numpy.array(encode(edges[-1]))))
def post_execute(self, result): """ Compute stats for the aggregated distributions and save the results on the datastore. """ loss_dt = self.oqparam.loss_dt() LI = len(loss_dt.names) dtlist = [('eid', U64), ('loss', (F32, LI))] R = self.R with self.monitor('saving outputs', autoflush=True): A = len(self.assetcol) # agg losses res = result['agg'] E, L = res.shape mean, std = scientific.mean_std(res) # shape L agglosses = numpy.zeros(L, stat_dt) agglosses['mean'] = F32(mean) agglosses['stddev'] = F32(std) # losses by asset losses_by_asset = numpy.zeros((A, R, L), stat_dt) for (l, r, aid, stat) in result['avg']: losses_by_asset[aid, r, l] = stat self.datastore['losses_by_asset'] = losses_by_asset self.datastore['agglosses'] = agglosses # losses by event lbe = numpy.fromiter(((ei, res[ei]) for ei in range(E)), dtlist) self.datastore['losses_by_event'] = lbe loss_types = ' '.join(self.oqparam.loss_dt().names) self.datastore.set_attrs('losses_by_event', loss_types=loss_types) # all losses if self.oqparam.asset_loss_table: array = numpy.zeros((A, E), loss_dt) for (l, r), losses_by_aid in result['all_losses'].items(): slc = self.event_slice(r) for aid in losses_by_aid: lba = losses_by_aid[aid] # E lt = loss_dt.names[l] array[lt][aid, slc] = lba self.datastore['asset_loss_table'] = array tags = [encode(tag) for tag in self.assetcol.tagcol] self.datastore.set_attrs('asset_loss_table', tags=tags)
def execute(self): """ Run in parallel `core_task(sources, sitecol, monitor)`, by parallelizing on the sources according to their weight and tectonic region type. """ oq = self.oqparam if oq.hazard_calculation_id and not oq.compare_with_classical: parent = util.read(self.oqparam.hazard_calculation_id) self.csm_info = parent['csm_info'] parent.close() self.calc_stats(parent) # post-processing return {} with self.monitor('managing sources', autoflush=True): smap = parallel.Starmap( self.core_task.__func__, monitor=self.monitor()) source_ids = [] data = [] for i, sources in enumerate(self._send_sources(smap)): source_ids.append(get_src_ids(sources)) for src in sources: # collect source data data.append((i, src.nsites, src.num_ruptures, src.weight)) if source_ids: self.datastore['task_sources'] = encode(source_ids) self.datastore.extend( 'source_data', numpy.array(data, source_data_dt)) self.calc_times = AccumDict(accum=numpy.zeros(2, F32)) try: acc = smap.reduce(self.agg_dicts, self.acc0()) self.store_rlz_info(acc.eff_ruptures) finally: with self.monitor('store source_info', autoflush=True): self.store_source_info(self.calc_times) if acc.nsites: if len(acc.nsites) > 50000: logging.warn( 'There are %d contributing sources', len(acc.nsites)) else: src_ids = sorted(acc.nsites) nsites = [acc.nsites[i] for i in src_ids] self.datastore['source_info'][src_ids, 'num_sites'] = nsites if not self.calc_times: raise RuntimeError('All sources were filtered away!') self.calc_times.clear() # save a bit of memory return acc
def post_execute(self, results): """ Save all the results of the disaggregation. NB: the number of results to save is #sites * #rlzs * #disagg_poes * #IMTs. :param results: a dictionary sid -> trti -> disagg matrix """ self.datastore.open('r+') T = len(self.trts) # build a dictionary sid -> 8D matrix of shape (T, ..., M, P) results = {sid: _8d_matrix(dic, T) for sid, dic in results.items()} # get the number of outputs shp = (len(self.sitecol), len(self.poes_disagg), len(self.imts)) logging.info('Extracting and saving the PMFs for %d outputs ' '(N=%s, P=%d, M=%d)', numpy.prod(shp), *shp) self.save_disagg_result(results, trts=encode(self.trts))
def postproc(self): """ Build aggregate loss curves in process """ dstore = self.datastore self.before_export() # set 'realizations' oq = self.oqparam stats = self.param['stats'] # store avg_losses-stats if oq.avg_losses: set_rlzs_stats(self.datastore, 'avg_losses') try: b = self.param['builder'] except KeyError: # don't build auxiliary tables return if dstore.parent: dstore.parent.open('r') # to read the ruptures if 'ruptures' in self.datastore and len(self.datastore['ruptures']): logging.info('Building loss tables') with self.monitor('building loss tables', measuremem=True): rlt, lbr = build_loss_tables(dstore) dstore['rup_loss_table'] = rlt dstore['losses_by_rlzi'] = lbr ridx = [rlt[:, lti].argmax() for lti in range(self.L)] dstore.set_attrs('rup_loss_table', ridx=ridx) logging.info('Building aggregate loss curves') with self.monitor('building agg_curves', measuremem=True): lbr = group_array(dstore['losses_by_event'][()], 'rlzi') dic = {r: arr['loss'] for r, arr in lbr.items()} array, arr_stats = b.build(dic, stats) loss_types = ' '.join(oq.loss_dt().names) units = self.datastore['cost_calculator'].get_units(loss_types.split()) if oq.individual_curves or self.R == 1: self.datastore['agg_curves-rlzs'] = array self.datastore.set_attrs( 'agg_curves-rlzs', return_periods=b.return_periods, loss_types=loss_types, units=units) if arr_stats is not None: self.datastore['agg_curves-stats'] = arr_stats self.datastore.set_attrs( 'agg_curves-stats', return_periods=b.return_periods, stats=[encode(name) for (name, func) in stats], loss_types=loss_types, units=units)
def reduce(fname, reduction_factor): """ Produce a submodel from `fname` by sampling the nodes randomly. Supports source models, site models and exposure models. As a special case, it is also able to reduce .csv files by sampling the lines. This is a debugging utility to reduce large computations to small ones. """ if fname.endswith('.csv'): with open(fname) as f: all_lines = f.readlines() lines = random_filter(all_lines, reduction_factor) shutil.copy(fname, fname + '.bak') print('Copied the original file in %s.bak' % fname) with open(fname, 'wb') as f: for line in lines: f.write(encode(line)) print('Extracted %d lines out of %d' % (len(lines), len(all_lines))) return node = nrml.read(fname) model = node[0] if model.tag.endswith('exposureModel'): total = len(model.assets) model.assets.nodes = random_filter(model.assets, reduction_factor) num_nodes = len(model.assets) elif model.tag.endswith('siteModel'): total = len(model) model.nodes = random_filter(model, reduction_factor) num_nodes = len(model) elif model.tag.endswith('sourceModel'): if node['xmlns'] != 'http://openquake.org/xmlns/nrml/0.5': raise InvalidFile('%s: not NRML0.5' % fname) total = sum(len(sg) for sg in model) num_nodes = 0 for sg in model: sg.nodes = random_filter(sg, reduction_factor) num_nodes += len(sg) else: raise RuntimeError('Unknown model tag: %s' % model.tag) shutil.copy(fname, fname + '.bak') print('Copied the original file in %s.bak' % fname) with open(fname, 'wb') as f: nrml.write([model], f, xmlns=node['xmlns']) print('Extracted %d nodes out of %d' % (num_nodes, total))
def set_rlzs_stats(dstore, prefix, arrayNR=None): """ :param dstore: a DataStore object :param prefix: dataset prefix :param arrayNR: an array of shape (N, R, ...) """ if arrayNR is None: # assume the -rlzs array is already stored arrayNR = dstore[prefix + '-rlzs'].value else: # store passed the -rlzs array dstore[prefix + '-rlzs'] = arrayNR R = arrayNR.shape[1] if R > 1: stats = dstore['oqparam'].hazard_stats() statnames, statfuncs = zip(*stats.items()) weights = dstore['csm_info'].rlzs['weight'] dstore[prefix + '-stats'] = compute_stats2(arrayNR, statfuncs, weights) dstore.set_attrs(prefix + '-stats', stats=encode(statnames))
def postproc(self): """ Build aggregate loss curves in process """ dstore = self.datastore oq = self.oqparam stats = self.param['stats'] # store avg_losses-stats if oq.avg_losses: set_rlzs_stats(self.datastore, 'avg_losses') try: b = self.param['builder'] except KeyError: # don't build auxiliary tables return if dstore.parent: dstore.parent.open('r') # to read the ruptures logging.info('Building loss tables') build_loss_tables(dstore) logging.info('Building aggregate loss curves') with self.monitor('building agg_curves', measuremem=True): lbr = group_array(dstore['losses_by_event'][()], 'rlzi') dic = {r: arr['loss'] for r, arr in lbr.items()} array, arr_stats = b.build(dic, stats) loss_types = oq.loss_dt().names units = self.datastore['cost_calculator'].get_units(loss_types) if oq.individual_curves or self.R == 1: self.datastore['agg_curves-rlzs'] = array # shape (P, R, L) self.datastore.set_attrs( 'agg_curves-rlzs', shape_descr=['return_periods', 'rlzs', 'loss_types'], return_periods=b.return_periods, rlzs=numpy.arange(self.R), loss_types=loss_types, units=units) if arr_stats is not None: self.datastore['agg_curves-stats'] = arr_stats # shape (P, S, L) self.datastore.set_attrs( 'agg_curves-stats', shape_descr=['return_periods', 'stats', 'loss_types'], return_periods=b.return_periods, stats=[encode(name) for (name, func) in stats], loss_types=loss_types, units=units)
def post_execute(self, results): """ Save all the results of the disaggregation. NB: the number of results to save is #sites * #rlzs * #disagg_poes * #IMTs. :param results: a dictionary (sid, rlzi, poe, imt) -> trti -> disagg matrix """ T = len(self.trts) # build a dictionary (sid, rlzi, poe, imt) -> 6D matrix results = {k: _to_matrix(v, T) for k, v in results.items()} # get the number of outputs shp = (len(self.sitecol), len(self.oqparam.poes_disagg or (None,)), len(self.oqparam.imtls)) # N, P, M logging.info('Extracting and saving the PMFs for %d outputs ' '(N=%s, P=%d, M=%d)', numpy.prod(shp), *shp) self.save_disagg_result(results, trts=encode(self.trts), num_ruptures=self.num_ruptures)
def set_rlzs_stats(dstore, prefix, arrayNR=None): """ :param dstore: a DataStore object :param prefix: dataset prefix :param arrayNR: an array of shape (N, R, ...) """ if arrayNR is None: # assume the -rlzs array is already stored arrayNR = dstore[prefix + '-rlzs'].value else: # store passed the -rlzs array dstore[prefix + '-rlzs'] = arrayNR R = arrayNR.shape[1] if R > 1: stats = dstore['oqparam'].hazard_stats() statnames, statfuncs = zip(*stats.items()) weights = dstore['weights'].value dstore[prefix + '-stats'] = compute_stats2(arrayNR, statfuncs, weights) dstore.set_attrs(prefix + '-stats', stats=encode(statnames))
def post_execute(self, results): """ Save all the results of the disaggregation. NB: the number of results to save is #sites * #rlzs * #disagg_poes * #IMTs. :param results: a dictionary of probability arrays """ # since an extremely small subset of the full disaggregation matrix # is saved this method can be run sequentially on the controller node logging.info('Extracting and saving the PMFs') for key, matrices in sorted(results.items()): sid, rlzi, poe, imt = key self.save_disagg_result(sid, matrices, rlzi, self.oqparam.investigation_time, imt, poe) self.datastore.set_attrs('disagg', trts=encode(self.trts), num_ruptures=self.num_ruptures)
def execute(self): """ Run in parallel `core_task(sources, sitecol, monitor)`, by parallelizing on the sources according to their weight and tectonic region type. """ oq = self.oqparam if oq.hazard_calculation_id and not oq.compare_with_classical: parent = datastore.read(self.oqparam.hazard_calculation_id) self.csm_info = parent['csm_info'] parent.close() self.calc_stats(parent) # post-processing return {} with self.monitor('managing sources', autoflush=True): smap = parallel.Starmap( self.core_task.__func__, monitor=self.monitor()) source_ids = [] data = [] for i, sources in enumerate(self._send_sources(smap)): source_ids.append(get_src_ids(sources)) for src in sources: # collect source data data.append((i, src.nsites, src.num_ruptures, src.weight)) if source_ids: self.datastore['task_sources'] = encode(source_ids) self.datastore.extend( 'source_data', numpy.array(data, source_data_dt)) self.calc_times = AccumDict(accum=numpy.zeros(2, F32)) try: acc = smap.reduce(self.agg_dicts, self.acc0()) self.store_rlz_info(acc.eff_ruptures) finally: with self.monitor('store source_info', autoflush=True): self.store_source_info(self.calc_times) if acc.nsites: src_ids = sorted(acc.nsites) nsites = [acc.nsites[i] for i in src_ids] self.datastore['source_info'][src_ids, 'num_sites'] = nsites if not self.calc_times: raise RuntimeError('All sources were filtered away!') self.calc_times.clear() # save a bit of memory return acc
def reduce(fname, reduction_factor): """ Produce a submodel from `fname` by sampling the nodes randomly. Supports source models, site models and exposure models. As a special case, it is also able to reduce .csv files by sampling the lines. This is a debugging utility to reduce large computations to small ones. """ if fname.endswith('.csv'): with open(fname) as f: all_lines = f.readlines() lines = random_filter(all_lines, reduction_factor) shutil.copy(fname, fname + '.bak') print('Copied the original file in %s.bak' % fname) with open(fname, 'wb') as f: for line in lines: f.write(encode(line)) print('Extracted %d lines out of %d' % (len(lines), len(all_lines))) return node = nrml.read(fname) model = node[0] if model.tag.endswith('exposureModel'): total = len(model.assets) model.assets.nodes = random_filter(model.assets, reduction_factor) num_nodes = len(model.assets) elif model.tag.endswith('siteModel'): total = len(model) model.nodes = random_filter(model, reduction_factor) num_nodes = len(model) elif model.tag.endswith('sourceModel'): total = len(model) model.nodes = random_filter(model, reduction_factor) num_nodes = len(model) else: raise RuntimeError('Unknown model tag: %s' % model.tag) shutil.copy(fname, fname + '.bak') print('Copied the original file in %s.bak' % fname) with open(fname, 'wb') as f: nrml.write([model], f, xmlns=node['xmlns']) print('Extracted %d nodes out of %d' % (num_nodes, total))
def post_execute(self, result): """ Saving loss curves in the datastore. :param result: aggregated result of the task classical_risk """ curve_res = {cp.loss_type: cp.curve_resolution for cp in self.riskmodel.curve_params if cp.user_provided} self.loss_curve_dt = scientific.build_loss_curve_dt( curve_res, insured_losses=False) ltypes = self.riskmodel.loss_types # loss curves stats are generated always stats = encode(list(self.oqparam.hazard_stats())) stat_curves = numpy.zeros((self.A, self.S), self.loss_curve_dt) avg_losses = numpy.zeros((self.A, self.S, self.L), F32) for l, a, losses, statpoes, statloss in result['stat_curves']: stat_curves_lt = stat_curves[ltypes[l]] for s in range(self.S): avg_losses[a, s, l] = statloss[s] base.set_array(stat_curves_lt['poes'][a, s], statpoes[s]) base.set_array(stat_curves_lt['losses'][a, s], losses) self.datastore['avg_losses-stats'] = avg_losses self.datastore.set_attrs('avg_losses-stats', stats=stats) self.datastore['loss_curves-stats'] = stat_curves self.datastore.set_attrs('loss_curves-stats', stats=stats) if self.R > 1: # individual realizations saved only if many loss_curves = numpy.zeros((self.A, self.R), self.loss_curve_dt) avg_losses = numpy.zeros((self.A, self.R, self.L), F32) for l, r, a, (losses, poes, avg) in result['loss_curves']: lc = loss_curves[a, r][ltypes[l]] avg_losses[a, r, l] = avg base.set_array(lc['losses'], losses) base.set_array(lc['poes'], poes) self.datastore['avg_losses-rlzs'] = avg_losses self.datastore['loss_curves-rlzs'] = loss_curves
def extract_aggregate(dstore, what): """ /extract/aggregate/avg_losses? kind=mean&loss_type=structural&tag=taxonomy&tag=occupancy """ name, qstring = what.split('?', 1) info = get_info(dstore) qdic = parse(qstring, info) suffix = '-rlzs' if qdic['rlzs'] else '-stats' tagnames = qdic.get('tag', []) assetcol = dstore['assetcol'] ltypes = qdic.get('loss_type', []) if ltypes: array = dstore[name + suffix][:, qdic['k'][0], ltypes[0]] else: array = dstore[name + suffix][:, qdic['k'][0]] aw = ArrayWrapper(assetcol.aggregate_by(tagnames, array), {}) for tagname in tagnames: setattr(aw, tagname, getattr(assetcol.tagcol, tagname)) aw.tagnames = encode(tagnames) if not ltypes: aw.extra = ('loss_type',) + tuple(info['loss_types']) return aw
sys.exit(2) def flag_set(section, setting): """True if the given boolean setting is enabled in openquake.cfg :param string section: name of the configuration file section :param string setting: name of the configuration file setting :returns: True if the setting is enabled in openquake.cfg, False otherwise """ setting = get(section, setting) if setting is None: return False return setting.lower() in ("true", "yes", "t", "1") def refresh(): """ Re-parse config files and refresh the cached configuration. NOTE: Use with caution. Calling this during some phases of a calculation could cause undesirable side-effects. """ cfg._load_from_file() port = int(get("dbserver", "port")) DBS_ADDRESS = (get("dbserver", "host"), port) DBS_AUTHKEY = encode(get("dbserver", "authkey"))
def __call__(self, *args): self.lst.append(b' '.join(encode(str(a)) for a in args))
def _save_csv(fname, lines, header): with open(fname, 'wb') as f: if header: f.write(encode(header)) for line in lines: f.write(encode(line))
def write_csv(dest, data, sep=',', fmt='%.6E', header=None, comment=None): """ :param dest: None, file, filename or io.BytesIO instance :param data: array to save :param sep: separator to use (default comma) :param fmt: formatting string (default '%12.8E') :param header: optional list with the names of the columns to display :param comment: optional comment dictionary """ if comment is not None: comment = ', '.join('%s=%r' % item for item in comment.items()) close = True if dest is None: # write on a temporary file fd, dest = tempfile.mkstemp(suffix='.csv') os.close(fd) if hasattr(dest, 'write'): # file-like object in append mode # it must be closed by client code close = False elif not hasattr(dest, 'getvalue'): # not a BytesIO, assume dest is a filename dest = open(dest, 'wb') try: # see if data is a composite numpy array data.dtype.fields except AttributeError: # not a composite array autoheader = [] else: autoheader = build_header(data.dtype) if comment: dest.write(encode('# %s\n' % comment)) someheader = header or autoheader if header != 'no-header' and someheader: dest.write(encode(sep.join(someheader) + u'\n')) if autoheader: all_fields = [col.split(':', 1)[0].split('~') for col in autoheader] for record in data: row = [] for fields in all_fields: val = extract_from(record, fields) if fields[0] in ('lon', 'lat', 'depth'): row.append('%.5f' % val) else: row.append(scientificformat(val, fmt)) dest.write(encode(sep.join(row) + u'\n')) else: for row in data: dest.write(encode(sep.join(scientificformat(col, fmt) for col in row) + u'\n')) if hasattr(dest, 'getvalue'): return dest.getvalue()[:-1] # a newline is strangely added elif close: dest.close() return dest.name
def save(self, fname): """Save the report""" with open(fname, 'wb') as f: f.write(encode(self.text))
sys.exit(2) def flag_set(section, setting): """True if the given boolean setting is enabled in openquake.cfg :param string section: name of the configuration file section :param string setting: name of the configuration file setting :returns: True if the setting is enabled in openquake.cfg, False otherwise """ setting = get(section, setting) if setting is None: return False return setting.lower() in ("true", "yes", "t", "1") def refresh(): """ Re-parse config files and refresh the cached configuration. NOTE: Use with caution. Calling this during some phases of a calculation could cause undesirable side-effects. """ cfg._load_from_file() port = int(get('dbserver', 'port')) DBS_ADDRESS = (get('dbserver', 'host'), port) DBS_AUTHKEY = encode(get('dbserver', 'authkey'))