def __fromh5__(self, array, attrs):
dd = collections.defaultdict(dict)
for (name_, literal_) in array:
name = decode(name_)
literal = decode(literal_)
if '.' in name:
k1, k2 = name.split('.', 1)
dd[k1][k2] = ast.literal_eval(literal)
else:
dd[name] = ast.literal_eval(literal)
vars(self).update(dd)
def get_serial(etag):
"""
>>> print(get_serial("grp=00~ses=0007~src=1-3~rup=018-01"))
18
"""
try:
trt, ses, src, rup = decode(etag).split('~')
except ValueError:
trt, ses, src, rup, sample = decode(etag).split('~')
serial = rup.split('=')[1].split('-')[0]
return int(serial)
def __fromh5__(self, dic, attrs):
for name in ('loss_types', 'deduc', 'i_lim', 'retro'):
setattr(self, name, [decode(x) for x in attrs[name].split()])
self.occupancy_periods = attrs['occupancy_periods']
self.time_event = attrs['time_event']
self.tot_sites = attrs['tot_sites']
self.nbytes = attrs['nbytes']
self.array = dic['array'].value
self.tagcol = dic['tagcol']
self.cost_calculator = dic['cost_calculator']
self.asset_refs = dic['asset_refs'].value
self.cost_calculator.tagi = {
decode(tagname): i for i, tagname in enumerate(self.tagnames)}
def check_obsolete_version(calculation_mode='WebUI'):
"""
Check if there is a newer version of the engine.
:param calculation_mode:
- the calculation mode when called from the engine
- an empty string when called from the WebUI
:returns:
- a message if the running version of the engine is obsolete
- the empty string if the engine is updated
- None if the check could not be performed (i.e. github is down)
"""
if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
# avoid flooding our API server with requests from CI systems
return
headers = {'User-Agent': 'OpenQuake Engine %s;%s;%s;%s' %
(__version__, calculation_mode, platform.platform(),
config.distribution.oq_distribute)}
try:
req = Request(OQ_API + '/engine/latest', headers=headers)
# NB: a timeout < 1 does not work
data = urlopen(req, timeout=1).read() # bytes
tag_name = json.loads(decode(data))['tag_name']
current = version_triple(__version__)
latest = version_triple(tag_name)
except Exception: # page not available or wrong version tag
return
if current < latest:
return ('Version %s of the engine is available, but you are '
'still using version %s' % (tag_name, __version__))
else:
return ''
def git_suffix(fname):
"""
:returns: `<short git hash>` if Git repository found
"""
# we assume that the .git folder is two levels above any package
# i.e. openquake/engine/../../.git
git_path = os.path.join(os.path.dirname(fname), '..', '..', '.git')
# macOS complains if we try to execute git and it's not available.
# Code will run, but a pop-up offering to install bloatware (Xcode)
# is raised. This is annoying in end-users installations, so we check
# if .git exists before trying to execute the git executable
if os.path.isdir(git_path):
try:
gh = subprocess.check_output(
['git', 'rev-parse', '--short', 'HEAD'],
stderr=open(os.devnull, 'w'),
cwd=os.path.dirname(git_path)).strip()
gh = "-git" + decode(gh) if gh else ''
return gh
except:
# trapping everything on purpose; git may not be installed or it
# may not work properly
pass
return ''
def __init__(self, amplification_group, magnitudes, distances):
"""
Setup the amplification factors.
:param amplification_group:
Amplification model as instance of :class:`h5py.Group`
:param magnitudes:
Array of magnitudes
:param distances:
Array of distances
"""
self.shape = None
self.periods = None
self.mean = None
self.sigma = None
self.magnitudes = magnitudes
self.distances = distances
self.parameter = decode(amplification_group.attrs["apply_to"])
self.values = numpy.array([float(key) for key in amplification_group])
self.argidx = numpy.argsort(self.values)
self.values = self.values[self.argidx]
if self.parameter in RuptureContext._slots_:
self.element = "Rupture"
elif self.parameter in site.site_param_dt:
self.element = "Sites"
else:
raise ValueError("Amplification parameter %s not recognised!"
% self.parameter)
self._build_data(amplification_group)
def _set_attrib(self, node, n, tn, v):
val = self.validators[tn]
try:
node.attrib[n] = val(decode(v))
except Exception as exc:
raise ValueError(
'Could not convert %s->%s: %s, line %s' %
(tn, val.__name__, exc, node.lineno))
def _display(node, indent, expandattrs, expandvals, output):
"""Core function to display a Node object"""
attrs = _displayattrs(node.attrib, expandattrs)
val = ' %s' % repr(node.text) \
if expandvals and node.text is not None else ''
output.write(decode((indent + striptag(node.tag) + attrs + val + '\n')))
for sub_node in node:
_display(sub_node, indent + ' ', expandattrs, expandvals, output)
def view_job_info(token, dstore):
"""
Determine the amount of data transferred from the controller node
to the workers and back in a classical calculation.
"""
job_info = h5py.File.__getitem__(dstore.hdf5, 'job_info')
rows = [(k, ast.literal_eval(decode(v))) for k, v in job_info]
return rst_table(rows)
def _humansize(literal):
dic = ast.literal_eval(decode(literal))
if isinstance(dic, dict):
items = sorted(dic.items(), key=operator.itemgetter(1), reverse=True)
lst = ['%s %s' % (k, humansize(v)) for k, v in items]
return ', '.join(lst)
else:
return str(dic)
def get_serial(etag):
"""
>>> print(get_serial("trt=00~ses=0007~src=1-3~rup=018-01"))
018
"""
trt, ses, src, rup = decode(etag).split('~')
serial = rup.split('=')[1].split('-')[0]
return serial
def __init__(self, dstore):
self.dstore = dstore
self.oq = oq = dstore['oqparam']
self.text = (decode(oq.description) + '\n' + '=' * len(oq.description))
info = dstore['job_info']
dpath = dstore.hdf5path
mtime = os.path.getmtime(dpath)
host = '%s:%s' % (info.hostname, decode(dpath))
updated = str(time.ctime(mtime))
versions = sorted(dstore['/'].attrs.items())
self.text += '\n\n' + views.rst_table([[host, updated]] + versions)
# NB: in the future, the sitecol could be transferred as
# an array by leveraging the HDF5 serialization protocol;
# for the moment however the size of the
# data to transfer is given by the usual pickle
sitecol_size = humansize(len(parallel.Pickled(dstore['sitecol'])))
self.text += '\n\nnum_sites = %d, sitecol = %s' % (
len(dstore['sitecol']), sitecol_size)
def to_str(self, expandattrs=True, expandvals=True):
"""
Convert the node into a string, intended for testing/debugging purposes
:param expandattrs:
print the values of the attributes if True, else print only the names
:param expandvals:
print the values if True, else print only the tag names
"""
out = io.BytesIO()
node_display(self, expandattrs, expandvals, out)
return decode(out.getvalue())
def _set_text(self, node, text, tag):
if text is None:
return
try:
val = self.validators[tag]
except KeyError:
return
try:
node.text = val(decode(text.strip()))
except Exception as exc:
raise ValueError('Could not convert %s->%s: %s' %
(tag, val.__name__, exc))
def __init__(self, dstore):
self.dstore = dstore
try:
self.builder = get_loss_builder(dstore)
except KeyError: # no 'events' for non event_based_risk
pass
self.assetcol = dstore['assetcol']
arefs = [decode(aref) for aref in self.assetcol.asset_refs]
self.str2asset = dict(zip(arefs, self.assetcol))
self.asset_refs = arefs
self.loss_types = dstore.get_attr('risk_model', 'loss_types')
self.R = dstore['csm_info'].get_num_rlzs()
def __init__(self, dstore):
self.dstore = dstore
self.oq = oq = dstore['oqparam']
self.text = (decode(oq.description) + '\n' + '=' * len(oq.description))
try:
num_rlzs = dstore['csm_info'].get_num_rlzs()
except KeyError:
num_rlzs = '?'
versions = sorted(dstore['/'].attrs.items())
self.text += '\n\n' + views.rst_table(versions)
self.text += '\n\nnum_sites = %d, num_levels = %d, num_rlzs = %s' % (
len(dstore['sitecol']), len(oq.imtls.array), num_rlzs)
def __repr__(self):
info_by_model = collections.OrderedDict()
for sm in self.source_models:
info_by_model[sm.path] = (
'_'.join(map(decode, sm.path)),
decode(sm.name),
[sg.id for sg in sm.src_groups],
sm.weight,
self.get_num_rlzs(sm))
summary = ['%s, %s, grp=%s, weight=%s: %d realization(s)' % ibm
for ibm in info_by_model.values()]
return '<%s\n%s>' % (
self.__class__.__name__, '\n'.join(summary))
def __toh5__(self):
# NB: the loss types do not contain spaces, so we can store them
# together as a single space-separated string
op = decode(self.occupancy_periods)
attrs = {'time_event': self.time_event or 'None',
'occupancy_periods': op,
'loss_types': ' '.join(self.loss_types),
'tot_sites': self.tot_sites,
'fields': ' '.join(self.fields),
'tagnames': encode(self.tagnames),
'nbytes': self.array.nbytes}
return dict(
array=self.array, tagcol=self.tagcol), attrs
def __fromh5__(self, dic, attrs):
sg_data = group_array(dic['sg_data'], 'sm_id')
sm_data = dic['sm_data']
vars(self).update(attrs)
self.gsim_fname = decode(self.gsim_fname)
if self.gsim_fname.endswith('.xml'):
self.gsim_lt = logictree.GsimLogicTree(
self.gsim_fname, sorted(self.trts))
else: # fake file with the name of the GSIM
self.gsim_lt = logictree.GsimLogicTree.from_(self.gsim_fname)
self.source_models = []
for sm_id, rec in enumerate(sm_data):
tdata = sg_data[sm_id]
srcgroups = [
sourceconverter.SourceGroup(
self.trts[trti], id=grp_id, eff_ruptures=effrup)
for grp_id, trti, effrup, sm_id in tdata if effrup]
path = tuple(str(decode(rec['path'])).split('_'))
trts = set(sg.trt for sg in srcgroups)
num_gsim_paths = self.gsim_lt.reduce(trts).get_num_paths()
sm = SourceModel(rec['name'], rec['weight'], path, srcgroups,
num_gsim_paths, sm_id, rec['samples'])
self.source_models.append(sm)
def git_suffix(fname):
"""
:returns: `<short git hash>` if Git repository found
"""
try:
gh = subprocess.check_output(
['git', 'rev-parse', '--short', 'HEAD'],
stderr=open(os.devnull, 'w'), cwd=os.path.dirname(fname)).strip()
gh = "-git" + decode(gh) if gh else ''
return gh
except:
# trapping everything on purpose; git may not be installed or it
# may not work properly
return ''
def __getitem__(self, path):
h5obj = super(File, self).__getitem__(path)
h5attrs = h5obj.attrs
if '__pyclass__' in h5attrs:
# NB: the `decode` below is needed for Python 3
modname, clsname = decode(h5attrs['__pyclass__']).rsplit('.', 1)
cls = getattr(importlib.import_module(modname), clsname)
obj = cls.__new__(cls)
if not hasattr(h5obj, 'shape'): # is group
h5obj = {unquote_plus(k): self['%s/%s' % (path, k)]
for k, v in h5obj.items()}
obj.__fromh5__(h5obj, h5attrs)
return obj
else:
return h5obj
def __toh5__(self):
# NB: the loss types do not contain spaces, so we can store them
# together as a single space-separated string
op = decode(self.occupancy_periods)
attrs = {'time_event': self.time_event or 'None',
'occupancy_periods': op,
'loss_types': ' '.join(self.loss_types),
'deduc': ' '.join(self.deduc),
'i_lim': ' '.join(self.i_lim),
'retro': ' '.join(self.retro),
'tot_sites': self.tot_sites,
'tagnames': encode(self.tagnames),
'nbytes': self.array.nbytes}
return dict(
array=self.array, cost_calculator=self.cost_calculator,
tagcol=self.tagcol, asset_refs=self.asset_refs), attrs
def __getitem__(self, path):
h5obj = super(File, self).__getitem__(path)
h5attrs = h5obj.attrs
if '__pyclass__' in h5attrs:
# NB: the `decode` below is needed for Python 3
cls = dotname2cls(decode(h5attrs['__pyclass__']))
obj = cls.__new__(cls)
if hasattr(h5obj, 'items'): # is group
h5obj = {unquote_plus(k): self['%s/%s' % (path, k)]
for k, v in h5obj.items()}
elif hasattr(h5obj, 'value'):
h5obj = h5obj.value
obj.__fromh5__(h5obj, h5attrs)
return obj
else:
return h5obj
def to_array(self, ebruptures):
data = []
for ebr in ebruptures:
rup = ebr.rupture
rc = self.cmaker.make_rupture_context(rup)
ruptparams = tuple(getattr(rc, param) for param in self.params)
point = rup.surface.get_middle_point()
multi_lons, multi_lats = rup.surface.get_surface_boundaries()
boundary = ','.join('((%s))' % ','.join(
'%.5f %.5f' % (lon, lat) for lon, lat in zip(lons, lats))
for lons, lats in zip(multi_lons, multi_lats))
try:
rate = ebr.rupture.occurrence_rate
except AttributeError: # for nonparametric sources
rate = numpy.nan
data.append((ebr.serial, ebr.multiplicity, len(ebr.sids),
rate, rup.mag, point.x, point.y, point.z,
rup.surface.get_strike(), rup.surface.get_dip(),
rup.rake, decode(boundary)) + ruptparams)
return numpy.array(data, self.dt)
def form(value):
"""
Format numbers in a nice way.
>>> form(0)
'0'
>>> form(0.0)
'0.0'
>>> form(0.0001)
'1.000E-04'
>>> form(1003.4)
'1,003'
>>> form(103.4)
'103'
>>> form(9.3)
'9.30000'
>>> form(-1.2)
'-1.2'
"""
if isinstance(value, FLOAT + INT):
if value <= 0:
return str(value)
elif value < .001:
return '%.3E' % value
elif value < 10 and isinstance(value, FLOAT):
return '%.5f' % value
elif value > 1000:
return '{:,d}'.format(int(round(value)))
elif numpy.isnan(value):
return 'NaN'
else: # in the range 10-1000
return str(int(value))
elif isinstance(value, bytes):
return decode(value)
elif isinstance(value, str):
return value
elif isinstance(value, numpy.object_):
return str(value)
elif hasattr(value, '__len__') and len(value) > 1:
return ' '.join(map(form, value))
return str(value)
def get_params(job_inis, **kw):
"""
Parse one or more INI-style config files.
:param job_inis:
List of configuration files (or list containing a single zip archive)
:param kw:
Optionally override some parameters
:returns:
A dictionary of parameters
"""
input_zip = None
if len(job_inis) == 1 and job_inis[0].endswith('.zip'):
input_zip = job_inis[0]
job_inis = extract_from_zip(
job_inis[0], ['job_hazard.ini', 'job_haz.ini',
'job.ini', 'job_risk.ini'])
not_found = [ini for ini in job_inis if not os.path.exists(ini)]
if not_found: # something was not found
raise IOError('File not found: %s' % not_found[0])
cp = configparser.ConfigParser()
cp.read(job_inis)
# directory containing the config files we're parsing
job_ini = os.path.abspath(job_inis[0])
base_path = decode(os.path.dirname(job_ini))
params = dict(base_path=base_path, inputs={'job_ini': job_ini})
if input_zip:
params['inputs']['input_zip'] = os.path.abspath(input_zip)
for sect in cp.sections():
_update(params, cp.items(sect), base_path)
_update(params, kw.items(), base_path) # override on demand
if params['inputs'].get('reqv'):
# using pointsource_distance=0 because of the reqv approximation
params['pointsource_distance'] = '0'
return params
def __fromh5__(self, dic, attrs):
# TODO: this is called more times than needed, maybe we should cache it
sg_data = group_array(dic['sg_data'], 'sm_id')
sm_data = dic['sm_data']
vars(self).update(attrs)
self.gsim_lt = dic['gsim_lt']
self.source_models = []
for sm_id, rec in enumerate(sm_data):
tdata = sg_data[sm_id]
srcgroups = [
sourceconverter.SourceGroup(
self.trts[data['trti']], id=data['grp_id'],
name=get_field(data, 'name', ''),
eff_ruptures=data['effrup'],
tot_ruptures=get_field(data, 'totrup', 0))
for data in tdata]
path = tuple(str(decode(rec['path'])).split('_'))
sm = logictree.LtSourceModel(
rec['name'], rec['weight'], path, srcgroups,
rec['num_rlzs'], sm_id, rec['samples'])
self.source_models.append(sm)
self.init()
def init(self, fle=None):
"""
Executes the preprocessing steps at the instantiation stage to read in
the tables from hdf5 and hold them in memory.
"""
if fle is None:
fname = self.kwargs.get('gmpe_table', self.GMPE_TABLE)
if fname is None:
raise ValueError('You forgot to set GMPETable.GMPE_TABLE!')
elif os.path.isabs(fname):
self.GMPE_TABLE = fname
else:
# NB: (hackish) GMPE_DIR must be set externally
self.GMPE_TABLE = os.path.abspath(
os.path.join(self.GMPE_DIR, fname))
fle = h5py.File(self.GMPE_TABLE, "r")
try:
# this is the format inside the datastore
self.distance_type = fle["distance_type"][()]
except KeyError:
# this is the original format outside the datastore
self.distance_type = decode(fle["Distances"].attrs["metric"])
self.REQUIRES_DISTANCES = set([self.distance_type])
# Load in magnitude
self.m_w = fle["Mw"][:]
# Load in distances
self.distances = fle["Distances"][:]
# Load intensity measure types and levels
self.imls = hdf_arrays_to_dict(fle["IMLs"])
self.DEFINED_FOR_INTENSITY_MEASURE_TYPES = set(self._supported_imts())
if "SA" in self.imls and "T" not in self.imls:
raise ValueError("Spectral Acceleration must be accompanied by "
"periods")
# Get the standard deviations
self._setup_standard_deviations(fle)
if "Amplification" in fle:
self._setup_amplification(fle)
def extract_agg_losses(dstore, what):
"""
Aggregate losses of the given loss type and tags. Use it as
/extract/agg_losses/structural?taxonomy=RC&zipcode=20126
/extract/agg_losses/structural?taxonomy=RC&zipcode=*
:returns:
an array of shape (T, R) if one of the tag names has a `*` value
an array of shape (R,), being R the number of realizations
an array of length 0 if there is no data for the given tags
"""
loss_type, tags = get_loss_type_tags(what)
if not loss_type:
raise ValueError('loss_type not passed in agg_losses/<loss_type>')
L = dstore['oqparam'].lti[loss_type]
if 'avg_losses-stats' in dstore:
stats = decode(dstore['avg_losses-stats'].attrs['stat'])
losses = dstore['avg_losses-stats'][:, :, L]
elif 'avg_losses-rlzs' in dstore:
stats = ['mean']
losses = dstore['avg_losses-rlzs'][:, :, L]
else:
raise KeyError('No losses found in %s' % dstore)
return _filter_agg(dstore['assetcol'], losses, tags, stats)
def export_damage(ekey, dstore):
loss_types = dstore.get_attr('composite_risk_model', 'loss_types')
damage_states = dstore.get_attr('composite_risk_model', 'damage_states')
rlzs = dstore['csm_info'].get_rlzs_assoc().realizations
dmg_by_asset = dstore['dmg_by_asset'] # shape (N, L, R)
assetcol = dstore['assetcol/array'].value
aref = dstore['asset_refs'].value
dmg_states = [DmgState(s, i) for i, s in enumerate(damage_states)]
D = len(dmg_states)
N, R = dmg_by_asset.shape
L = len(loss_types)
fnames = []
for l, r in itertools.product(range(L), range(R)):
lt = loss_types[l]
rlz = rlzs[r]
dd_asset = []
for n, ass in enumerate(assetcol):
assref = decode(aref[ass['idx']])
dist = dmg_by_asset[n, r][lt]
site = Site(ass['lon'], ass['lat'])
for ds in range(D):
dd_asset.append(
DmgDistPerAsset(ExposureData(assref, site), dmg_states[ds],
dist['mean'][ds], dist['stddev'][ds]))
f1 = export_dmg_xml(('dmg_dist_per_asset', 'xml'), dstore, dmg_states,
dd_asset, lt, rlz)
max_damage = dmg_states[-1]
# the collapse map is extracted from the damage distribution per asset
# (dda) by taking the value corresponding to the maximum damage
collapse_map = [dda for dda in dd_asset if dda.dmg_state == max_damage]
f2 = export_dmg_xml(('collapse_map', 'xml'), dstore, dmg_states,
collapse_map, lt, rlz)
fnames.extend(sum((f1 + f2).values(), []))
return sorted(fnames)
def to_array(self, ebruptures):
"""
Convert a list of ebruptures into an array of dtype RuptureRata.dt
"""
data = []
for ebr in ebruptures:
rup = ebr.rupture
self.cmaker.add_rup_params(rup)
ruptparams = tuple(getattr(rup, param) for param in self.params)
point = rup.surface.get_middle_point()
multi_lons, multi_lats = rup.surface.get_surface_boundaries()
bounds = ','.join('((%s))' % ','.join(
'%.5f %.5f' % (lon, lat) for lon, lat in zip(lons, lats))
for lons, lats in zip(multi_lons, multi_lats))
try:
rate = ebr.rupture.occurrence_rate
except AttributeError: # for nonparametric sources
rate = numpy.nan
data.append(
(ebr.serial, ebr.srcidx, ebr.n_occ, rate,
rup.mag, point.x, point.y, point.z, rup.surface.get_strike(),
rup.surface.get_dip(), rup.rake,
'MULTIPOLYGON(%s)' % decode(bounds)) + ruptparams)
return numpy.array(data, self.dt)
def to_array(self, ebruptures):
"""
Convert a list of ebruptures into an array of dtype RuptureRata.dt
"""
data = []
for ebr in ebruptures:
rup = ebr.rupture
self.cmaker.add_rup_params(rup)
ruptparams = tuple(getattr(rup, param) for param in self.params)
point = rup.surface.get_middle_point()
multi_lons, multi_lats = rup.surface.get_surface_boundaries()
bounds = ','.join('((%s))' % ','.join(
'%.5f %.5f' % (lon, lat) for lon, lat in zip(lons, lats))
for lons, lats in zip(multi_lons, multi_lats))
try:
rate = ebr.rupture.occurrence_rate
except AttributeError: # for nonparametric sources
rate = numpy.nan
data.append(
(ebr.serial, ebr.multiplicity, ebr.eidx1, rate,
rup.mag, point.x, point.y, point.z, rup.surface.get_strike(),
rup.surface.get_dip(), rup.rake,
'MULTIPOLYGON(%s)' % decode(bounds)) + ruptparams)
return numpy.array(data, self.dt)
def __getitem__(self, aid):
a = self.array[aid]
values = {
lt: a['value-' + lt]
for lt in self.loss_types if lt != 'occupants'
}
if 'occupants' in self.array.dtype.names:
values['occupants_' + str(self.time_event)] = a['occupants']
return Asset(
a['idx'],
[a[decode(name)] for name in self.tagnames],
number=a['number'],
location=(
valid.longitude(a['lon']), # round coordinates
valid.latitude(a['lat'])),
values=values,
area=a['area'],
deductibles={lt[self.D:]: a[lt]
for lt in self.deduc},
insurance_limits={lt[self.I:]: a[lt]
for lt in self.i_lim},
retrofitted=a['retrofitted'] if self.retro else None,
calc=self.cost_calculator,
ordinal=aid)
def export_loss_maps_stats_xml_geojson(ekey, dstore):
loss_maps = get_loss_maps(dstore, 'stats')
N, S = loss_maps.shape
assetcol = dstore['assetcol/array'].value
aref = dstore['asset_refs'].value
fnames = []
export_type = ekey[1]
writercls = (risk_writers.LossMapGeoJSONWriter if export_type == 'geojson'
else risk_writers.LossMapXMLWriter)
for writer, (ltype, poe, s,
insflag) in _gen_writers(dstore, writercls, ekey[0]):
ins = '_ins' if insflag else ''
if ltype not in loss_maps.dtype.names:
continue
array = loss_maps[ltype + ins][:, s]
curves = []
poe_str = 'poe-%s' % poe
for ass, val in zip(assetcol, array[poe_str]):
loc = Location(ass['lon'], ass['lat'])
curve = LossMap(loc, decode(aref[ass['idx']]), val, None)
curves.append(curve)
writer.serialize(curves)
fnames.append(writer._dest)
return sorted(fnames)
def view_task_hazard(token, dstore):
"""
Display info about a given task. Here are a few examples of usage::
$ oq show task_hazard:0 # the fastest task
$ oq show task_hazard:-1 # the slowest task
"""
tasks = set(dstore['task_info'])
if 'source_data' not in dstore:
return 'Missing source_data'
if 'classical_split_filter' in tasks:
data = dstore['task_info/classical_split_filter'][()]
else:
data = dstore['task_info/compute_gmfs'][()]
data.sort(order='duration')
rec = data[int(token.split(':')[1])]
taskno = rec['taskno']
arr = get_array(dstore['source_data'][()], taskno=taskno)
st = [stats('nsites', arr['nsites']), stats('weight', arr['weight'])]
sources = dstore['task_sources'][taskno - 1].split()
srcs = set(decode(s).split(':', 1)[0] for s in sources)
res = 'taskno=%d, weight=%d, duration=%d s, sources="%s"\n\n' % (
taskno, rec['weight'], rec['duration'], ' '.join(sorted(srcs)))
return res + rst_table(st, header='variable mean stddev min max n'.split())
def export_bcr_map(ekey, dstore):
assetcol = dstore['assetcol/array'].value
aref = dstore['asset_refs'].value
bcr_data = dstore['bcr-rlzs']
N, R = bcr_data.shape
realizations = dstore['csm_info'].get_rlzs_assoc().realizations
loss_types = dstore.get_attr('composite_risk_model', 'loss_types')
fnames = []
writer = writers.CsvWriter(fmt=writers.FIVEDIGITS)
for rlz in realizations:
for l, loss_type in enumerate(loss_types):
rlz_data = bcr_data[loss_type][:, rlz.ordinal]
path = dstore.build_fname('bcr-%s' % loss_type, rlz, 'csv')
data = [[
'lon', 'lat', 'asset_ref', 'average_annual_loss_original',
'average_annual_loss_retrofitted', 'bcr'
]]
for ass, value in zip(assetcol, rlz_data):
data.append((ass['lon'], ass['lat'], decode(aref[ass['idx']]),
value['annual_loss_orig'],
value['annual_loss_retro'], value['bcr']))
writer.save(data, path)
fnames.append(path)
return writer.getsaved()
def view_task_hazard(token, dstore):
"""
Display info about a given task. Here are a few examples of usage::
$ oq show task_hazard:0 # the fastest task
$ oq show task_hazard:-1 # the slowest task
"""
tasks = set(dstore['task_info'])
if 'source_data' not in dstore:
return 'Missing source_data'
if 'classical_split_filter' in tasks:
data = dstore['task_info/classical_split_filter'][()]
else:
data = dstore['task_info/compute_gmfs'][()]
data.sort(order='duration')
rec = data[int(token.split(':')[1])]
taskno = rec['taskno']
arr = get_array(dstore['source_data'][()], taskno=taskno)
st = [stats('nsites', arr['nsites']), stats('weight', arr['weight'])]
sources = dstore['task_sources'][taskno - 1].split()
srcs = set(decode(s).split(':', 1)[0] for s in sources)
res = 'taskno=%d, weight=%d, duration=%d s, sources="%s"\n\n' % (
taskno, rec['weight'], rec['duration'], ' '.join(sorted(srcs)))
return res + rst_table(st, header='variable mean stddev min max n'.split())
def __fromh5__(self, dic, attrs):
# TODO: this is called more times than needed, maybe we should cache it
sg_data = group_array(dic['sg_data'], 'sm_id')
sm_data = dic['sm_data']
vars(self).update(attrs)
self.gsim_lt = dic['gsim_lt']
self.source_models = []
for sm_id, rec in enumerate(sm_data):
tdata = sg_data[sm_id]
srcgroups = [
sourceconverter.SourceGroup(self.trts[data['trti']],
id=data['grp_id'],
name=get_field(data, 'name', ''),
eff_ruptures=data['effrup'],
tot_ruptures=get_field(
data, 'totrup', 0))
for data in tdata
]
path = tuple(str(decode(rec['path'])).split('_'))
sm = logictree.LtSourceModel(rec['name'], rec['weight'], path,
srcgroups, rec['num_rlzs'], sm_id,
rec['samples'])
self.source_models.append(sm)
self.init()
def check_obsolete_version(calculation_mode='WebUI'):
"""
Check if there is a newer version of the engine.
:param calculation_mode:
- the calculation mode when called from the engine
- an empty string when called from the WebUI
:returns:
- a message if the running version of the engine is obsolete
- the empty string if the engine is updated
- None if the check could not be performed (i.e. github is down)
"""
if os.environ.get('JENKINS_URL') or os.environ.get('TRAVIS'):
# avoid flooding our API server with requests from CI systems
return
headers = {
'User-Agent':
'OpenQuake Engine %s;%s;%s;%s' %
(__version__, calculation_mode, platform.platform(),
config.distribution.oq_distribute)
}
try:
req = Request(OQ_API + '/engine/latest', headers=headers)
# NB: a timeout < 1 does not work
data = urlopen(req, timeout=1).read() # bytes
tag_name = json.loads(decode(data))['tag_name']
current = version_triple(__version__)
latest = version_triple(tag_name)
except Exception: # page not available or wrong version tag
return
if current < latest:
return ('Version %s of the engine is available, but you are '
'still using version %s' % (tag_name, __version__))
else:
return ''
def __fromh5__(self, array, attrs):
vars(self).update(attrs)
self.cost_types = dict(zip(self.loss_types, array['cost_type']))
self.area_types = dict(zip(self.loss_types, array['area_type']))
self.units = dict(zip(self.loss_types, decode(array['unit'])))
def __fromh5__(self, dic, attrs):
self.tagnames = [decode(name) for name in attrs['tagnames']]
for tagname in dic:
setattr(self, tagname + '_idx',
{tag: idx for idx, tag in enumerate(dic[tagname])})
setattr(self, tagname, dic[tagname].value)
def _get(dstore, name):
try:
dset = dstore[name + '-stats']
return dset, decode(dset.attrs['stat'])
except KeyError: # single realization
return dstore[name + '-rlzs'], ['mean']
def capitalize(words):
"""
Capitalize words separated by spaces.
"""
return ' '.join(w.capitalize() for w in decode(words).split(' '))
def export_disagg_csv(ekey, dstore):
oq = dstore['oqparam']
sitecol = dstore['sitecol']
hmap4 = dstore['hmap4']
rlzs = dstore['full_lt'].get_realizations()
best_rlzs = dstore['best_rlzs'][:]
N, M, P, Z = hmap4.shape
imts = list(oq.imtls)
fnames = []
bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()}
ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d'
if ekey[0] == 'disagg_traditional':
ex += '&traditional=1'
trad = '-traditional'
else:
trad = ''
skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT')
metadata = dstore.metadata
poes_disagg = ['nan'] * P
for p in range(P):
try:
poes_disagg[p] = str(oq.poes_disagg[p])
except IndexError:
pass
for s in range(N):
rlzcols = ['rlz%d' % r for r in best_rlzs[s]]
lon, lat = sitecol.lons[s], sitecol.lats[s]
weights = numpy.array([rlzs[r].weight['weight'] for r in best_rlzs[s]])
weights /= weights.sum() # normalize to 1
metadata.update(investigation_time=oq.investigation_time,
mag_bin_edges=bins['Mag'].tolist(),
dist_bin_edges=bins['Dist'].tolist(),
lon_bin_edges=bins['Lon'][s].tolist(),
lat_bin_edges=bins['Lat'][s].tolist(),
eps_bin_edges=bins['Eps'].tolist(),
tectonic_region_types=decode(bins['TRT'].tolist()),
rlz_ids=best_rlzs[s].tolist(),
weights=weights.tolist(),
lon=lon, lat=lat)
for k in oq.disagg_outputs:
splits = k.lower().split('_')
header = ['imt', 'poe'] + splits + rlzcols
values = []
nonzeros = []
for m, p in iproduct(M, P):
imt = imts[m]
aw = extract(dstore, ex % (k, imt, s, p))
# for instance for Mag_Dist [(mag, dist, poe0, poe1), ...]
poes = aw[:, len(splits):]
if 'trt' in header:
nonzeros.append(True)
else:
nonzeros.append(poes.any()) # nonzero poes
for row in aw:
values.append([imt, poes_disagg[p]] + list(row))
if any(nonzeros):
com = {key: value for key, value in metadata.items()
if value is not None and key not in skip_keys}
com.update(metadata)
fname = dstore.export_path('%s%s-%d.csv' % (k, trad, s))
writers.write_csv(fname, values, header=header,
comment=com, fmt='%.5E')
fnames.append(fname)
return sorted(fnames)
def export_disagg_csv_xml(ekey, dstore):
oq = dstore['oqparam']
sitecol = dstore['sitecol']
hmap4 = dstore['hmap4']
N, M, P, Z = hmap4.shape
imts = list(oq.imtls)
rlzs = dstore['full_lt'].get_realizations()
fnames = []
writercls = hazard_writers.DisaggXMLWriter
bins = {name: dset[:] for name, dset in dstore['disagg-bins'].items()}
ex = 'disagg?kind=%s&imt=%s&site_id=%s&poe_id=%d&z=%d'
skip_keys = ('Mag', 'Dist', 'Lon', 'Lat', 'Eps', 'TRT')
for s, m, p, z in iproduct(N, M, P, Z):
dic = {
k: dstore['disagg/' + k][s, m, p, ..., z]
for k in oq.disagg_outputs
}
if sum(arr.sum() for arr in dic.values()) == 0: # no data
continue
imt = from_string(imts[m])
r = hmap4.rlzs[s, z]
rlz = rlzs[r]
iml = hmap4[s, m, p, z]
poe_agg = dstore['poe4'][s, m, p, z]
fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d.xml' %
(r, imt, s, p))
lon, lat = sitecol.lons[s], sitecol.lats[s]
metadata = dstore.metadata
metadata.update(investigation_time=oq.investigation_time,
imt=imt.name,
smlt_path='_'.join(rlz.sm_lt_path),
gsimlt_path=rlz.gsim_rlz.pid,
lon=lon,
lat=lat,
mag_bin_edges=bins['Mag'].tolist(),
dist_bin_edges=bins['Dist'].tolist(),
lon_bin_edges=bins['Lon'][s].tolist(),
lat_bin_edges=bins['Lat'][s].tolist(),
eps_bin_edges=bins['Eps'].tolist(),
tectonic_region_types=decode(bins['TRT'].tolist()))
if ekey[1] == 'xml':
metadata['sa_period'] = getattr(imt, 'period', None) or None
metadata['sa_damping'] = getattr(imt, 'damping', None)
writer = writercls(fname, **metadata)
data = []
for k in oq.disagg_outputs:
data.append(DisaggMatrix(poe_agg, iml, k.split('_'), dic[k]))
writer.serialize(data)
fnames.append(fname)
else: # csv
metadata['poe'] = poe_agg
for k in oq.disagg_outputs:
header = k.lower().split('_') + ['poe']
com = {
key: value
for key, value in metadata.items()
if value is not None and key not in skip_keys
}
com.update(metadata)
fname = dstore.export_path('rlz-%d-%s-sid-%d-poe-%d_%s.csv' %
(r, imt, s, p, k))
values = extract(dstore, ex % (k, imt, s, p, z))
writers.write_csv(fname,
values,
header=header,
comment=com,
fmt='%.5E')
fnames.append(fname)
return sorted(fnames)
def get_tag(self, tagname, tagidx):
"""
:returns: the tag associated to the given tagname and tag index
"""
return '%s=%s' % (tagname, decode(getattr(self, tagname)[tagidx]))
def gen_tags(self, tagname):
"""
:yields: the tags associated to the given tagname
"""
for tagvalue in getattr(self, tagname):
yield '%s=%s' % (tagname, decode(tagvalue))
