def _save_uhs(job, uhs_results, poe, rlz=None, statistics=None, quantile=None):
"""
Save computed UHS data to the DB.
UHS results can be either for an end branch or for mean or quantile
statistics.
:param job:
:class:`openquake.engine.db.models.OqJob` instance to be associated
with the results.
:param uhs_results:
UHS computation results structured like the output of :func:`make_uhs`.
:param float poe:
Probability of exceedance of the hazard maps from which these UH
Spectra were produced.
:param rlz:
:class:`openquake.engine.db.models.LtRealization`. Specify only if
these results are for an end branch.
:param statistics:
'mean' or 'quantile'. Specify only if these are statistical results.
:param float quantile:
Specify only if ``statistics`` == 'quantile'.
"""
output = models.Output(oq_job=job, output_type='uh_spectra')
uhs = models.UHS(
poe=poe,
investigation_time=job.hazard_calculation.investigation_time,
periods=uhs_results['periods'],
)
if rlz is not None:
uhs.lt_realization = rlz
output.display_name = _UHS_DISP_NAME_FMT % dict(poe=poe, rlz=rlz.id)
elif statistics is not None:
uhs.statistics = statistics
if statistics == 'quantile':
uhs.quantile = quantile
output.display_name = (_UHS_DISP_NAME_QUANTILE_FMT %
dict(poe=poe, quantile=quantile))
else:
# mean
output.display_name = _UHS_DISP_NAME_MEAN_FMT % dict(poe=poe)
output.save()
uhs.output = output
# This should fail if neither `lt_realization` nor `statistics` is defined:
uhs.save()
with transaction.commit_on_success(using='reslt_writer'):
inserter = CacheInserter(models.UHSData, CURVE_CACHE_SIZE)
for lon, lat, imls in uhs_results['uh_spectra']:
inserter.add(
models.UHSData(uhs_id=uhs.id,
imls='{%s}' % ','.join(str(x) for x in imls),
location='POINT(%s %s)' % (lon, lat)))
inserter.flush()
def test_insert_gmf(self):
cache = CacheInserter(GmfData, 10)
gmf1 = GmfData(gmf_id=1, imt='PGA', gmvs=[], rupture_ids=[], site_id=1)
gmf2 = GmfData(gmf_id=1, imt='PGA', gmvs=[], rupture_ids=[], site_id=2)
cache.add(gmf1)
cache.add(gmf2)
cache.flush()
connection = writer.connections['reslt_writer']
self.assertEqual(
connection.data,
'1\t\\N\tPGA\t\\N\t\\N\t{}\t{}\t1\n1\t\\N\tPGA\t\\N\t\\N\t{}\t{}\t2\n'
)
self.assertEqual(connection.table, '"hzrdr"."gmf_data"')
self.assertEqual(connection.columns, [
'gmf_id', 'ses_id', 'imt', 'sa_period', 'sa_damping', 'gmvs',
'rupture_ids', 'site_id'
])
class EnginePerformanceMonitor(PerformanceMonitor):
"""
Performance monitor specialized for the engine. It takes in input a
string, a job_id, and a celery task; the on_exit method
send the relevant info to the uiapi.performance table.
For efficiency reasons the saving on the database is delayed and
done in chunks of 1,000 rows each. That means that hundreds of
concurrents task can log simultaneously on the uiapi.performance table
without problems. You can save more often by calling the .cache.flush()
method; it is automatically called for you by the oqtask decorator;
it is also called at the end of the main engine process.
"""
# globals per process
cache = CacheInserter(models.Performance, 1000) # store at most 1k objects
pgpid = None
pypid = None
@classmethod
def store_task_id(cls, job_id, task):
with cls('storing task id', job_id, task, flush=True):
pass
@classmethod
def monitor(cls, method):
"""
A decorator to add monitoring to calculator methods. The only
constraints are:
1) the method has no arguments except self
2) there is an attribute self.job.id
"""
def newmeth(self):
with cls(method.__name__, self.job.id, flush=True):
return method(self)
newmeth.__name__ = method.__name__
return newmeth
def __init__(self,
operation,
job_id,
task=None,
tracing=False,
profile_pymem=True,
profile_pgmem=False,
flush=False):
self.operation = operation
self.job_id = job_id
if task:
self.task = task
self.task_id = task.request.id
else:
self.task = None
self.task_id = None
self.tracing = tracing
self.profile_pymem = profile_pymem
self.profile_pgmem = profile_pgmem
self.flush = flush
if self.profile_pymem and self.pypid is None:
self.__class__.pypid = os.getpid()
if self.profile_pgmem and self.pgpid is None:
# this may be slow
pgpid = connections['job_init'].cursor().\
connection.get_backend_pid()
try:
psutil.Process(pgpid)
except psutil.error.NoSuchProcess: # db on a different machine
pass
else:
self.__class__.pgpid = pgpid
if tracing:
self.tracer = logs.tracing(operation)
super(EnginePerformanceMonitor,
self).__init__([self.pypid, self.pgpid])
def copy(self, operation):
"""
Return a copy of the monitor usable for a different operation
in the same task.
"""
return self.__class__(operation, self.job_id, self.task, self.tracing,
self.profile_pymem, self.profile_pgmem)
def on_exit(self):
"""
Save the memory consumption on the uiapi.performance table.
"""
n_measures = len(self.mem)
if n_measures == 2:
pymemory, pgmemory = self.mem
elif n_measures == 1:
pymemory, = self.mem
pgmemory = None
elif n_measures == 0: # profile_pymem was False
pymemory = pgmemory = None
else:
raise ValueError('Got %d memory measurements, must be <= 2' %
n_measures)
if self.exc is None: # save only valid calculations
perf = models.Performance(oq_job_id=self.job_id,
task_id=self.task_id,
task=getattr(self.task, '__name__',
None),
operation=self.operation,
start_time=self.start_time,
duration=self.duration,
pymemory=pymemory,
pgmemory=pgmemory)
self.cache.add(perf)
if self.flush:
self.cache.flush()
def __enter__(self):
super(EnginePerformanceMonitor, self).__enter__()
if self.tracing:
self.tracer.__enter__()
return self
def __exit__(self, etype, exc, tb):
super(EnginePerformanceMonitor, self).__exit__(etype, exc, tb)
if self.tracing:
self.tracer.__exit__(etype, exc, tb)
