def _get_calcs(request_get_dict, user_name, id=None): # TODO if superuser with should show all the calculations i.e. # helper to get job+calculation data from the oq-engine database job_params = oqe_models.JobParam.objects.filter( name='description', job__user_name=user_name).order_by('-id') if id is not None: job_params = job_params.filter(job_id=id) if 'job_type' in request_get_dict: job_type = request_get_dict.get('job_type') job_params = job_params.filter( job__hazard_calculation__isnull=job_type == 'hazard') if 'is_running' in request_get_dict: is_running = request_get_dict.get('is_running') job_params = job_params.filter( job__is_running=valid.boolean(is_running)) if 'relevant' in request_get_dict: relevant = request_get_dict.get('relevant') job_params = job_params.filter(job__relevant=valid.boolean(relevant)) return [(jp.job.id, jp.job.status, jp.job.job_type, jp.job.is_running, jp.value) for jp in job_params]
def _get_calcs(request_get_dict, user_name, user_is_super=False, id=None): # TODO if superuser with should show all the calculations i.e. # helper to get job+calculation data from the oq-engine database job_params = oqe_models.JobParam.objects.filter( name='description').order_by('-id') if not user_is_super: job_params = job_params.filter(job__user_name=user_name) if id is not None: job_params = job_params.filter(job_id=id) if 'job_type' in request_get_dict: job_type = request_get_dict.get('job_type') job_params = job_params.filter( job__hazard_calculation__isnull=job_type == 'hazard') if 'is_running' in request_get_dict: is_running = request_get_dict.get('is_running') job_params = job_params.filter( job__is_running=valid.boolean(is_running)) if 'relevant' in request_get_dict: relevant = request_get_dict.get('relevant') job_params = job_params.filter(job__relevant=valid.boolean(relevant)) return [(jp.job.id, jp.job.user_name, jp.job.status, jp.job.job_type, jp.job.is_running, jp.value) for jp in job_params]
def initialize_sources(self): """ Parse source models, apply uncertainties and validate source logic trees. Save in the database LtSourceModel and TrtModel objects. """ logs.LOG.progress("initializing sources") parallel_source_splitting = valid.boolean( config.get('hazard', 'parallel_source_splitting') or 'false') self.composite_model = readinput.get_composite_source_model( self.oqparam, self.site_collection, no_distribute=not parallel_source_splitting) for sm in self.composite_model: # create an LtSourceModel for each distinct source model lt_model = models.LtSourceModel.objects.create( hazard_calculation=self.job, sm_lt_path=self.tilepath + sm.path, ordinal=sm.ordinal, sm_name=sm.name, weight=sm.weight, samples=sm.samples) self._source_models.append(lt_model) gsims_by_trt = sm.gsim_lt.values # save TrtModels for each tectonic region type # and stored the db ID in the in-memory models for trt_mod in sm.trt_models: trt_mod.id = models.TrtModel.objects.create( lt_model=lt_model, tectonic_region_type=trt_mod.trt, num_sources=len(trt_mod), num_ruptures=trt_mod.num_ruptures, min_mag=trt_mod.min_mag, max_mag=trt_mod.max_mag, gsims=gsims_by_trt[trt_mod.trt]).id # rebuild the info object with the trt_ids coming from the db self.composite_model.info = source.CompositionInfo( self.composite_model.source_model_lt, self.composite_model.source_models)
def float_or_flag(value, isAbsolute=None): """ Validate the attributes/tags insuranceLimit and deductible """ if isAbsolute is None: # considering the insuranceLimit attribute return valid.positivefloat(value) else: return valid.boolean(isAbsolute)
def _get_calcs(request_get_dict, user_name, user_is_super=False, id=None): # helper to get job+calculation data from the oq-engine database jobs = oqe_models.OqJob.objects.filter() if not user_is_super: jobs = jobs.filter(user_name=user_name) if id is not None: jobs = jobs.filter(id=id) if 'job_type' in request_get_dict: job_type = request_get_dict.get('job_type') jobs = jobs.filter(hazard_calculation__isnull=job_type == 'hazard') if 'is_running' in request_get_dict: is_running = request_get_dict.get('is_running') jobs = jobs.filter(is_running=valid.boolean(is_running)) if 'relevant' in request_get_dict: relevant = request_get_dict.get('relevant') jobs = jobs.filter(relevant=valid.boolean(relevant)) return [(job.id, job.user_name, job.status, job.job_type, job.is_running, job.description) for job in jobs.order_by('-id')]
def check_nodes(self): """ Check that the expected celery nodes are all up. The loop continues until the main thread keeps running. """ while self.job_is_running(sleep=self.interval): live_nodes = self.ping(timeout=self.interval) if live_nodes < self.live_nodes: dead_nodes = list(self.live_nodes - live_nodes) logs.LOG.critical( 'Cluster nodes not accessible: %s', dead_nodes) terminate = boolean( config.get('celery', 'terminate_job_when_celery_is_down')) if terminate: os.kill(os.getpid(), signal.SIGABRT) # commit suicide
from openquake import risklib from openquake import nrmllib from openquake.commonlib import readini, valid INPUT_TYPES = set(dict(models.INPUT_TYPE_CHOICES)) UNABLE_TO_DEL_HC_FMT = "Unable to delete hazard calculation: %s" UNABLE_TO_DEL_RC_FMT = "Unable to delete risk calculation: %s" LOG_FORMAT = ( "[%(asctime)s %(job_type)s job #%(job_id)s %(hostname)s " "%(levelname)s %(processName)s/%(process)s] %(message)s" ) TERMINATE = valid.boolean(config.get("celery", "terminate_workers_on_revoke")) def cleanup_after_job(job, terminate): """ Release the resources used by an openquake job. In particular revoke the running tasks (if any). :param int job_id: the job id :param bool terminate: the celery revoke command terminate flag """ # Using the celery API, terminate and revoke and terminate any running # tasks associated with the current job. task_ids = Performance.objects.filter(oq_job=job, operation="storing task id", task_id__isnull=False).values_list( "task_id", flat=True )
from openquake.engine.writer import CacheInserter from openquake.engine.settings import DATABASES from openquake.engine.db.models import Performance from openquake.engine.db.schema.upgrades import upgrader from openquake import hazardlib, risklib, commonlib from openquake.commonlib import readinput, valid INPUT_TYPES = set(dict(models.INPUT_TYPE_CHOICES)) UNABLE_TO_DEL_HC_FMT = 'Unable to delete hazard calculation: %s' UNABLE_TO_DEL_RC_FMT = 'Unable to delete risk calculation: %s' TERMINATE = valid.boolean(config.get('celery', 'terminate_workers_on_revoke')) class InvalidHazardCalculationID(Exception): pass RISK_HAZARD_MAP = dict( scenario_risk='scenario', scenario_damage='scenario', classical_risk='classical', classical_bcr='classical', event_based_risk='event_based', event_based_bcr='event_based') def cleanup_after_job(job, terminate):
from openquake.engine.performance import EnginePerformanceMonitor from openquake.engine.writer import CacheInserter from openquake.engine.settings import DATABASES from openquake.engine.db.models import Performance from openquake.engine.db.schema.upgrades import upgrader from openquake import hazardlib, risklib, commonlib from openquake.commonlib import readinput, valid INPUT_TYPES = set(dict(models.INPUT_TYPE_CHOICES)) UNABLE_TO_DEL_HC_FMT = 'Unable to delete hazard calculation: %s' UNABLE_TO_DEL_RC_FMT = 'Unable to delete risk calculation: %s' TERMINATE = valid.boolean(config.get('celery', 'terminate_workers_on_revoke')) class InvalidHazardCalculationID(Exception): pass RISK_HAZARD_MAP = dict(scenario_risk=['scenario'], scenario_damage=['scenario'], classical_risk=['classical'], classical_bcr=['classical'], classical_damage=['classical'], event_based_risk=['event_based'], event_based_bcr=['event_based'])
def test_boolean(self): self.assertEqual(valid.boolean('0'), False) self.assertEqual(valid.boolean('1'), True) self.assertEqual(valid.boolean('false'), False) self.assertEqual(valid.boolean('true'), True) with self.assertRaises(ValueError): valid.boolean('') with self.assertRaises(ValueError): valid.boolean('xxx') with self.assertRaises(ValueError): valid.boolean('True') with self.assertRaises(ValueError): valid.boolean('False')
# You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. """Utility functions related to splitting work into tasks.""" import types from openquake.baselib.performance import Monitor from openquake.commonlib import parallel, valid from openquake.engine import logs from openquake.engine.utils import config litetask = parallel.litetask celery_queue = config.get('amqp', 'celery_queue') SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit')) HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit')) USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false') parallel.check_mem_usage.__defaults__ = ( Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT) if USE_CELERY: from celery.result import ResultSet from celery.app import current_app from celery.task import task class OqTaskManager(parallel.TaskManager): """ A celery-based task manager. The usage is:: oqm = OqTaskManager(do_something, logs.LOG.progress) oqm.send(arg1, arg2) oqm.send(arg3, arg4)
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. """Engine: A collection of fundamental functions for initializing and running calculations.""" import sys import traceback from openquake.baselib.performance import Monitor from openquake.commonlib import valid from openquake.commonlib.oqvalidation import OqParam from openquake.calculators import base from openquake.engine import logs from openquake.engine.utils import config, tasks TERMINATE = valid.boolean( config.get('celery', 'terminate_workers_on_revoke') or 'false') USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false') if USE_CELERY: import celery.task.control def set_concurrent_tasks_default(): """ Set the default for concurrent_tasks to twice the number of workers. Returns the number of live celery nodes (i.e. the number of machines). """ stats = celery.task.control.inspect(timeout=1).stats() if not stats: sys.exit("No live compute nodes, aborting calculation") num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
# # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>. """Utility functions related to splitting work into tasks.""" import types from openquake.baselib.performance import Monitor from openquake.commonlib import parallel, valid from openquake.engine import logs from openquake.engine.utils import config litetask = parallel.litetask celery_queue = config.get('amqp', 'celery_queue') SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit')) HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit')) USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false') parallel.check_mem_usage.__defaults__ = (Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT) if USE_CELERY: from celery.result import ResultSet from celery.app import current_app from celery.task import task class OqTaskManager(parallel.TaskManager): """ A celery-based task manager. The usage is:: oqm = OqTaskManager(do_something, logs.LOG.progress) oqm.send(arg1, arg2) oqm.send(arg3, arg4)