Example #1
0
 def test_get_with_empty_section_data(self):
     # config.get() returns `None` if the section data dict is empty
     with patch('openquake.engine.config.get_section') as mock:
         mock.return_value = dict()
         self.assertTrue(config.get("whatever", "key") is None)
         self.assertEqual(1, mock.call_count)
         self.assertEqual([("whatever",), {}], mock.call_args)
Example #2
0
 def test_get_with_unknown_key(self):
     """config.get() returns `None` if the `key` is not known."""
     with patch('openquake.engine.config.get_section') as mock:
         mock.return_value = dict(b=1)
         self.assertTrue(config.get("arghh", "c") is None)
         self.assertEqual(1, mock.call_count)
         self.assertEqual([("arghh", ), {}], mock.call_args)
Example #3
0
 def test_get_with_empty_section_data(self):
     # config.get() returns `None` if the section data dict is empty
     with patch('openquake.engine.config.get_section') as mock:
         mock.return_value = dict()
         self.assertTrue(config.get("whatever", "key") is None)
         self.assertEqual(1, mock.call_count)
         self.assertEqual([("whatever", ), {}], mock.call_args)
Example #4
0
def dbserver(cmd):
    """
    start/stop/restart the database server, or return its status
    """
    if valid.boolean(config.get('dbserver', 'multi_user')):
        sys.exit('oq dbserver only works in single user mode')

    status = get_status()
    if cmd == 'status':
        print(status)
    elif cmd == 'stop':
        if status == 'running':
            logs.dbcmd('stop')
            print('stopped')
        else:
            print('already stopped')
    elif cmd == 'start':
        if status == 'not-running':
            runserver()
            print('started')
        else:
            print('already running')
    elif cmd == 'restart':
        if status == 'running':
            logs.dbcmd('stop')
            print('stopped')
        runserver()
        print('started')
Example #5
0
 def test_get_with_unknown_key(self):
     """config.get() returns `None` if the `key` is not known."""
     with patch('openquake.engine.config.get_section') as mock:
         mock.return_value = dict(b=1)
         self.assertTrue(config.get("arghh", "c") is None)
         self.assertEqual(1, mock.call_count)
         self.assertEqual([("arghh",), {}], mock.call_args)
Example #6
0
 def test_get_with_nonempty_section_data_and_known_key(self):
     # config.get() correctly returns the configuration datum for known
     # sections/keys
     with patch('openquake.engine.config.get_section') as mock:
         mock.return_value = dict(a=11)
         self.assertEqual(11, config.get("hmmm", "a"))
         self.assertEqual(1, mock.call_count)
         self.assertEqual([("hmmm", ), {}], mock.call_args)
Example #7
0
 def test_get_with_nonempty_section_data_and_known_key(self):
     # config.get() correctly returns the configuration datum for known
     # sections/keys
     with patch('openquake.engine.config.get_section') as mock:
         mock.return_value = dict(a=11)
         self.assertEqual(11, config.get("hmmm", "a"))
         self.assertEqual(1, mock.call_count)
         self.assertEqual([("hmmm",), {}], mock.call_args)
Example #8
0
def webui(cmd, hostport='127.0.0.1:8800'):
    """
    start the webui server in foreground or perform other operation on the
    django application
    """
    dbstatus = get_status()
    if dbstatus == 'not-running':
        if valid.boolean(config.get('dbserver', 'multi_user')):
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        dbserver.runserver()

    if cmd == 'start':
        rundjango('runserver', hostport)
    elif cmd == 'syncdb':
        rundjango('syncdb')
Example #9
0
def ensure_on():
    """
    Start the DbServer if it is off
    """
    if get_status() == 'not-running':
        if valid.boolean(config.get('dbserver', 'multi_user')):
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'])

        # wait for the dbserver to start
        waiting_seconds = 10
        while get_status() == 'not-running':
            if waiting_seconds == 0:
                sys.exit('The DbServer cannot be started after 10 seconds. '
                         'Please check the configuration')
            time.sleep(1)
            waiting_seconds -= 1
Example #10
0
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""Engine: A collection of fundamental functions for initializing and running
calculations."""

import sys
import signal
import traceback

from openquake.baselib.performance import Monitor
from openquake.commonlib import valid, parallel
from openquake.commonlib.oqvalidation import OqParam
from openquake.calculators import base
from openquake.engine import logs, config

TERMINATE = valid.boolean(
    config.get('celery', 'terminate_workers_on_revoke') or 'false')

USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')

if USE_CELERY:
    import celery.task.control

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks to twice the number of workers.
        Returns the number of live celery nodes (i.e. the number of machines).
        """
        stats = celery.task.control.inspect(timeout=1).stats()
        if not stats:
            sys.exit("No live compute nodes, aborting calculation")
        num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
Example #11
0
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
from openquake.baselib.performance import Monitor
from openquake.risklib import valid
from openquake.engine import config

SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit'))
HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit'))
USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')

if USE_CELERY:
    os.environ['OQ_DISTRIBUTE'] = 'celery'

# NB: this import must go AFTER the setting of OQ_DISTRIBUTE
from openquake.commonlib import parallel

parallel.check_mem_usage.__defaults__ = (
    Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT)


def confirm(prompt):
    """
Example #12
0
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import importlib

from openquake.baselib import sap
from openquake.commonlib import __version__
from openquake import commands

from openquake.risklib import valid
from openquake.engine import config

USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')

# the environment variable has the precedence over the configuration file
if 'OQ_DISTRIBUTE' not in os.environ and USE_CELERY:
    os.environ['OQ_DISTRIBUTE'] = 'celery'

# force cluster users to use `oq engine` so that we have centralized logs
if USE_CELERY and 'run' in sys.argv:
    sys.exit('You are on a cluster and you are using oq run?? '
             'Use oq engine --run instead!')

def oq():
    modnames = ['openquake.commands.%s' % mod[:-3]
                for mod in os.listdir(commands.__path__[0])
                if mod.endswith('.py') and not mod.startswith('_')]
    for modname in modnames:
Example #13
0
import os
import sys
import signal
import traceback

from openquake.baselib.performance import Monitor
from openquake.risklib import valid
from openquake.commonlib import parallel, readinput
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib import export, datastore
from openquake.calculators import base, views
from openquake.engine import logs, config

TERMINATE = valid.boolean(
    config.get('celery', 'terminate_workers_on_revoke') or 'false')

USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')

if USE_CELERY:
    import celery.task.control

    def set_concurrent_tasks_default():
        """
        Set the default for concurrent_tasks.
        Returns the number of live celery nodes (i.e. the number of machines).
        """
        stats = celery.task.control.inspect(timeout=1).stats()
        if not stats:
            sys.exit("No live compute nodes, aborting calculation")
        num_cores = sum(stats[k]['pool']['max-concurrency'] for k in stats)
Example #14
0
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, version_db, what_if_I_upgrade,
           run_hazard, run_risk, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info'):
    """
    Run a calculation using the traditional command line API
    """
    config.abort_if_no_config_available()

    if run or run_hazard or run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = os.path.abspath(
            os.path.expanduser(config_file))
        config.refresh()

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    if not os.path.exists(datastore.DATADIR):
        os.makedirs(datastore.DATADIR)

    # check if the DbServer is up
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        err = sock.connect_ex(config.DBS_ADDRESS)
    finally:
        sock.close()
    if err:
        multi_user = valid.boolean(config.get('dbserver', 'multi_user'))
        if multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'])
    if upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if version_db:
        print(logs.dbcmd('version_db'))
        sys.exit(0)

    if what_if_I_upgrade:
        print(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id:
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        job_inis = map(os.path.expanduser, run.split(','))
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], log_level,
                             log_file, exports)
            # run risk
            run_job(job_inis[1], log_level, log_file,
                    exports, hazard_calculation_id=job_id)
        else:
            run_job(
                os.path.expanduser(run), log_level, log_file,
                exports, hazard_calculation_id=hc_id)
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            print(line)
    elif run_hazard is not None:
        print('WARN: --rh/--run-hazard are deprecated, use --run instead',
              file=sys.stderr)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(os.path.expanduser(run_hazard), log_level,
                log_file, exports)
    elif delete_calculation is not None:
        delete_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            print(line)
    elif run_risk is not None:
        print('WARN: --rr/--run-risk are deprecated, use --run instead',
              file=sys.stderr)
        if hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(
            os.path.expanduser(run_risk),
            log_level, log_file, exports,
            hazard_calculation_id=hc_id)

    # export
    elif make_html_report:
        print('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            print(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            print(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports):
            print(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports):
            print(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        parser.parentparser.prog = 'oq engine'
        parser.parentparser.print_usage()
Example #15
0
from __future__ import print_function
import io
import os
import getpass
import logging

from openquake.hazardlib.calc.hazard_curve import zero_curves
from openquake.baselib import sap
from openquake.risklib import scientific, valid
from openquake.commonlib import datastore
from openquake.commonlib.writers import write_csv
from openquake.commonlib.util import rmsep
from openquake.engine import config, logs
from openquake.calculators.views import view

MULTI_USER = valid.boolean(config.get('dbserver', 'multi_user') or 'false')
if MULTI_USER:
    # get the datastore of the user who ran the job
    def read(calc_id):
        job = logs.dbcmd('get_job', calc_id, getpass.getuser())
        datadir = os.path.dirname(job.ds_calc_dir)
        return datastore.read(job.id, datadir=datadir)
else:  # get the datastore of the current user
    read = datastore.read


def get_hcurves_and_means(dstore):
    """
    Extract hcurves from the datastore and compute their means.

    :returns: curves_by_rlz, mean_curves
Example #16
0
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
from openquake.baselib.performance import Monitor
from openquake.risklib import valid
from openquake.engine import config

SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit'))
HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit'))
USE_CELERY = valid.boolean(config.get('celery', 'use_celery') or 'false')

if USE_CELERY:
    os.environ['OQ_DISTRIBUTE'] = 'celery'

# NB: this import must go AFTER the setting of OQ_DISTRIBUTE
from openquake.commonlib import parallel

parallel.check_mem_usage.__defaults__ = (Monitor(), SOFT_MEM_LIMIT,
                                         HARD_MEM_LIMIT)


def confirm(prompt):
    """
Example #17
0
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

from openquake.baselib.performance import Monitor
from openquake.commonlib import parallel
from openquake.engine import config

SOFT_MEM_LIMIT = int(config.get('memory', 'soft_mem_limit'))
HARD_MEM_LIMIT = int(config.get('memory', 'hard_mem_limit'))

parallel.check_mem_usage.__defaults__ = (
    Monitor(), SOFT_MEM_LIMIT, HARD_MEM_LIMIT)


def confirm(prompt):
    """
    Ask for confirmation, given a ``prompt`` and return a boolean value.
    """
    while True:
        try:
            answer = raw_input(prompt)
        except KeyboardInterrupt:
            # the user presses ctrl+c, just say 'no'
Example #18
0
def main():
    arg_parser = set_up_arg_parser()

    args = arg_parser.parse_args()

    exports = args.exports or 'xml,csv'

    if args.version:
        print openquake.engine.__version__
        sys.exit(0)

    if args.run or args.run_hazard or args.run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if args.config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = \
            abspath(expanduser(args.config_file))
        config.refresh()

    if args.no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the DbServer is up
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        err = sock.connect_ex(config.DBS_ADDRESS)
    finally:
        sock.close()
    if err:
        multi_user = valid.boolean(config.get('dbserver', 'multi_user'))
        if multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        dblog = os.path.expanduser('~/oq-dbserver.log')
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'], stderr=open(dblog, 'w'))
    if args.upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif args.yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if args.version_db:
        print logs.dbcmd('version_db')
        sys.exit(0)

    if args.what_if_I_upgrade:
        print logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if args.hazard_calculation_id:
        hc_id = get_job_id(args.hazard_calculation_id)
    else:
        hc_id = None
    if args.run:
        job_inis = map(expanduser, args.run.split(','))
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % args.run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], args.log_level,
                             log_file, args.exports)
            # run risk
            run_job(job_inis[1], args.log_level, log_file,
                    args.exports, hazard_calculation_id=job_id)
        else:
            run_job(
                expanduser(args.run), args.log_level, log_file,
                args.exports, hazard_calculation_id=hc_id)
    # hazard
    elif args.list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            print line
    elif args.run_hazard is not None:
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(expanduser(args.run_hazard), args.log_level,
                log_file, args.exports)
    elif args.delete_calculation is not None:
        delete_calculation(args.delete_calculation, args.yes)
    # risk
    elif args.list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            print line
    elif args.run_risk is not None:
        if args.hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(
            expanduser(args.run_risk),
            args.log_level, log_file, args.exports,
            hazard_calculation_id=hc_id)

    # export
    elif args.make_html_report:
        print 'Written', make_report(args.make_html_report)
        sys.exit(0)

    elif args.list_outputs is not None:
        hc_id = get_job_id(args.list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            print line
    elif args.show_view is not None:
        job_id, view_name = args.show_view
        print views.view(view_name, datastore.read(int(job_id)))
    elif args.show_log is not None:
        hc_id = get_job_id(args.show_log[0])
        for line in logs.dbcmd('get_log', hc_id):
            print line

    elif args.export_output is not None:
        output_id, target_dir = args.export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, expanduser(target_dir), exports):
            print line

    elif args.export_outputs is not None:
        job_id, target_dir = args.export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, expanduser(target_dir), exports):
            print line

    elif args.delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
    else:
        arg_parser.print_usage()