Example #1
0
    def test_export_aggregate_loss(self):
        writer = 'csv.writer'

        self.output_mock.aggregate_loss.id = 0
        self.output_mock.aggregate_loss.mean = 1
        self.output_mock.aggregate_loss.std_dev = 2
        self.output_mock.aggregate_loss.loss_type = "structural"
        self.output_mock.output_type = 'aggregate_loss'

        with mock.patch(writer) as m:
            ret = core.export_output(('aggregate_loss', 'csv'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([], m.writerow.call_args_list)
            self.assertEqual("/tmp/aggregate-loss-0.csv", ret)
Example #2
0
    def test_export_aggregate_loss(self):
        writer = 'csv.writer'

        self.output_mock.aggregate_loss.id = 0
        self.output_mock.aggregate_loss.mean = 1
        self.output_mock.aggregate_loss.std_dev = 2
        self.output_mock.aggregate_loss.loss_type = "structural"
        self.output_mock.output_type = 'aggregate_loss'

        with mock.patch(writer) as m:
            ret = core.export_output(('aggregate_loss', 'csv'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([], m.writerow.call_args_list)
            self.assertEqual("/tmp/aggregate-loss-0.csv", ret)
Example #3
0
    def test_export_agg_loss_curve(self):
        writer = 'openquake.commonlib.risk_writers.AggregateLossCurveXMLWriter'

        self.output_mock.loss_curve.id = 0
        self.output_mock.loss_curve.loss_type = "structural"
        self.output_mock.output_type = 'agg_loss_curve'
        with mock.patch(writer) as m:
            ret = core.export_output(('agg_loss_curve', 'xml'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([(('/tmp/loss-curves-0.xml', ),
                              {'gsim_tree_path': None,
                               'investigation_time': 30,
                               'quantile_value': None,
                               'source_model_tree_path': None,
                               'statistics': 'mean',
                               'unit': 'bucks',
                               'loss_type': 'structural'})], m.call_args_list)
            self.assertEqual('/tmp/loss-curves-0.xml', ret)
Example #4
0
    def test_export_agg_loss_curve(self):
        writer = 'openquake.commonlib.risk_writers.AggregateLossCurveXMLWriter'

        self.output_mock.loss_curve.id = 0
        self.output_mock.loss_curve.loss_type = "structural"
        self.output_mock.output_type = 'agg_loss_curve'
        with mock.patch(writer) as m:
            ret = core.export_output(('agg_loss_curve', 'xml'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([(('/tmp/loss-curves-0.xml', ), {
                'gsim_tree_path': None,
                'investigation_time': 30,
                'quantile_value': None,
                'source_model_tree_path': None,
                'statistics': 'mean',
                'unit': 'bucks',
                'loss_type': 'structural'
            })], m.call_args_list)
            self.assertEqual('/tmp/loss-curves-0.xml', ret)
Example #5
0
    def test_export_bcr_distribution(self):
        writer = 'openquake.commonlib.risk_writers.BCRMapXMLWriter'

        self.output_mock.bcr_distribution.id = 0
        self.output_mock.bcr_distribution.loss_type = "structural"
        self.output_mock.output_type = 'bcr_distribution'

        with mock.patch(writer) as m:
            ret = core.export_output(('bcr_distribution', 'xml'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([(('/tmp/bcr-distribution-0.xml', ),
                              {'asset_life_expectancy': 10,
                               'gsim_tree_path': None,
                               'interest_rate': 0.3,
                               'quantile_value': None,
                               'source_model_tree_path': None,
                               'statistics': 'mean',
                               'unit': 'bucks',
                               'loss_type': 'structural'})], m.call_args_list)
            self.assertEqual('/tmp/bcr-distribution-0.xml', ret)
Example #6
0
    def test_export_bcr_distribution(self):
        writer = 'openquake.commonlib.risk_writers.BCRMapXMLWriter'

        self.output_mock.bcr_distribution.id = 0
        self.output_mock.bcr_distribution.loss_type = "structural"
        self.output_mock.output_type = 'bcr_distribution'

        with mock.patch(writer) as m:
            ret = core.export_output(('bcr_distribution', 'xml'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([(('/tmp/bcr-distribution-0.xml', ), {
                'asset_life_expectancy': 10,
                'gsim_tree_path': None,
                'interest_rate': 0.3,
                'quantile_value': None,
                'source_model_tree_path': None,
                'statistics': 'mean',
                'unit': 'bucks',
                'loss_type': 'structural'
            })], m.call_args_list)
            self.assertEqual('/tmp/bcr-distribution-0.xml', ret)
Example #7
0
    def test_export_loss_map(self):
        writer = 'openquake.commonlib.risk_writers.LossMapXMLWriter'

        self.output_mock.loss_map.id = 0
        self.output_mock.loss_map.poe = 0.1
        self.output_mock.loss_map.loss_type = "structural"
        self.output_mock.output_type = 'loss_map'

        with mock.patch(writer) as m:
            ret = core.export_output(('loss_map', 'xml'),
                                     self.output_mock, "/tmp/")

            self.assertEqual([(('/tmp/loss-maps-0.xml', ),
                              {'gsim_tree_path': None,
                               'investigation_time': 30,
                               'loss_category': 'air',
                               'poe': 0.1,
                               'quantile_value': None,
                               'source_model_tree_path': None,
                               'statistics': 'mean',
                               'unit': 'bucks',
                               'loss_type': 'structural'})], m.call_args_list)
            self.assertEqual('/tmp/loss-maps-0.xml', ret)
Example #8
0
    def test_export_loss_map(self):
        writer = 'openquake.commonlib.risk_writers.LossMapXMLWriter'

        self.output_mock.loss_map.id = 0
        self.output_mock.loss_map.poe = 0.1
        self.output_mock.loss_map.loss_type = "structural"
        self.output_mock.output_type = 'loss_map'

        with mock.patch(writer) as m:
            ret = core.export_output(('loss_map', 'xml'), self.output_mock,
                                     "/tmp/")

            self.assertEqual([(('/tmp/loss-maps-0.xml', ), {
                'gsim_tree_path': None,
                'investigation_time': 30,
                'loss_category': 'air',
                'poe': 0.1,
                'quantile_value': None,
                'source_model_tree_path': None,
                'statistics': 'mean',
                'unit': 'bucks',
                'loss_type': 'structural'
            })], m.call_args_list)
            self.assertEqual('/tmp/loss-maps-0.xml', ret)
Example #9
0
def main(no_distribute=False,
         yes=False,
         upgrade_db=False,
         db_version=False,
         what_if_I_upgrade=False,
         list_hazard_calculations=False,
         list_risk_calculations=False,
         delete_uncompleted_calculations=False,
         multi=False,
         reuse_input=False,
         *,
         log_file=None,
         make_html_report=None,
         run=None,
         delete_calculation: int = None,
         hazard_calculation_id: int = None,
         list_outputs: int = None,
         show_log=None,
         export_output=None,
         export_outputs=None,
         param='',
         config_file=None,
         exports='',
         log_level='info'):
    """
    Run a calculation using the traditional command line API
    """
    if not run:
        # configure a basic logging
        logs.init()

    if config_file:
        config.read(os.path.abspath(os.path.expanduser(config_file)),
                    soft_mem_limit=int,
                    hard_mem_limit=int,
                    port=int,
                    multi_user=valid.boolean,
                    serialize_jobs=valid.boolean,
                    strict=valid.boolean,
                    code=exec)

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    datadir = datastore.get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    dbserver.ensure_on()
    # check if we are talking to the right server
    err = dbserver.check_foreign()
    if err:
        sys.exit(err)

    if upgrade_db:
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if db_version:
        safeprint(logs.dbcmd('db_version'))
        sys.exit(0)

    if what_if_I_upgrade:
        safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id == -1:
        # get the latest calculation of the current user
        hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
    elif hazard_calculation_id:
        # make it possible to use calculations made by another user
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        pars = dict(p.split('=', 1) for p in param.split(',')) if param else {}
        if reuse_input:
            pars['cachedir'] = datadir
        if hc_id:
            pars['hazard_calculation_id'] = str(hc_id)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        job_inis = [os.path.expanduser(f) for f in run]
        pars['multi'] = multi
        run_jobs(job_inis, log_level, log_file, exports, **pars)

    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd('list_calculations', 'hazard',
                               getpass.getuser()):
            safeprint(line)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            safeprint(line)

    # export
    elif make_html_report:
        safeprint('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            safeprint(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            safeprint(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(dskey, calc_id, datadir,
                                       os.path.expanduser(target_dir), exports
                                       or DEFAULT_EXPORTS):
            safeprint(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
                                        exports or DEFAULT_EXPORTS):
            safeprint(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
    else:
        print("Please pass some option, see oq engine --help")
Example #10
0
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, db_version, what_if_I_upgrade,
           run_hazard, run_risk, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info'):
    """
    Run a calculation using the traditional command line API
    """
    if run or run_hazard or run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if config_file:
        config.load(os.path.abspath(os.path.expanduser(config_file)))

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    datadir = datastore.get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    dbserver.ensure_on()
    # check if we are talking to the right server
    err = dbserver.check_foreign()
    if err:
        sys.exit(err)

    if upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if db_version:
        safeprint(logs.dbcmd('db_version'))
        sys.exit(0)

    if what_if_I_upgrade:
        safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id:
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        job_ini = os.path.expanduser(run)
        open(job_ini, 'rb').read()  # IOError if the file does not exist
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(os.path.expanduser(run), log_level, log_file,
                exports, hazard_calculation_id=hc_id)
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            safeprint(line)
    elif run_hazard is not None:
        safeprint('WARN: --rh/--run-hazard are deprecated, use --run instead',
                  file=sys.stderr)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(os.path.expanduser(run_hazard), log_level,
                log_file, exports)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            safeprint(line)
    elif run_risk is not None:
        safeprint('WARN: --rr/--run-risk are deprecated, use --run instead',
                  file=sys.stderr)
        if hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(
            os.path.expanduser(run_risk),
            log_level, log_file, exports,
            hazard_calculation_id=hc_id)

    # export
    elif make_html_report:
        safeprint('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            safeprint(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            safeprint(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports or 'csv,xml'):
            safeprint(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
            safeprint(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()
Example #11
0
def engine(log_file,
           no_distribute,
           yes,
           config_file,
           make_html_report,
           upgrade_db,
           db_version,
           what_if_I_upgrade,
           run,
           list_hazard_calculations,
           list_risk_calculations,
           delete_calculation,
           delete_uncompleted_calculations,
           hazard_calculation_id,
           list_outputs,
           show_log,
           export_output,
           export_outputs,
           exports='',
           log_level='info',
           reuse_hazard=False):
    """
    Run a calculation using the traditional command line API
    """
    if not run:
        # configure a basic logging
        logs.init()

    if config_file:
        config.read(os.path.abspath(os.path.expanduser(config_file)),
                    soft_mem_limit=int,
                    hard_mem_limit=int,
                    port=int,
                    multi_user=valid.boolean,
                    multi_node=valid.boolean)

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    datadir = datastore.get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    dbserver.ensure_on()
    # check if we are talking to the right server
    err = dbserver.check_foreign()
    if err:
        sys.exit(err)

    if upgrade_db:
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if db_version:
        safeprint(logs.dbcmd('db_version'))
        sys.exit(0)

    if what_if_I_upgrade:
        safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id == -1:
        # get the latest calculation of the current user
        hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
    elif hazard_calculation_id:
        # make it possible to use calculations made by another user
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        job_inis = [os.path.expanduser(f) for f in run]
        if len(job_inis) == 1 and not hc_id:
            # init logs before calling get_oqparam
            logs.init('nojob', getattr(logging, log_level.upper()))
            # not using logs.handle that logs on the db
            oq = readinput.get_oqparam(job_inis[0])
            smart_run(job_inis[0], oq, log_level, log_file, exports,
                      reuse_hazard)
            return
        for i, job_ini in enumerate(job_inis):
            open(job_ini, 'rb').read()  # IOError if the file does not exist
            job_id = run_job(job_ini,
                             log_level,
                             log_file,
                             exports,
                             hazard_calculation_id=hc_id)
            if not hc_id:  # use the first calculation as base for the others
                hc_id = job_id
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd('list_calculations', 'hazard',
                               getpass.getuser()):
            safeprint(line)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            safeprint(line)

    # export
    elif make_html_report:
        safeprint('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            safeprint(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            safeprint(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(dskey, calc_id, datadir,
                                       os.path.expanduser(target_dir), exports
                                       or 'csv,xml'):
            safeprint(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(hc_id, os.path.expanduser(target_dir),
                                        exports or 'csv,xml'):
            safeprint(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()
Example #12
0
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, version_db, what_if_I_upgrade,
           run_hazard, run_risk, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info'):
    """
    Run a calculation using the traditional command line API
    """
    config.abort_if_no_config_available()

    if run or run_hazard or run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = os.path.abspath(
            os.path.expanduser(config_file))
        config.refresh()

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    if not os.path.exists(datastore.DATADIR):
        os.makedirs(datastore.DATADIR)

    dbserver.ensure_on()
    
    if upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if version_db:
        print(logs.dbcmd('version_db'))
        sys.exit(0)

    if what_if_I_upgrade:
        print(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id:
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        job_inis = [os.path.expanduser(ini) for ini in run.split(',')]
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], log_level,
                             log_file, exports)
            # run risk
            run_job(job_inis[1], log_level, log_file,
                    exports, hazard_calculation_id=job_id)
        else:
            run_job(
                os.path.expanduser(run), log_level, log_file,
                exports, hazard_calculation_id=hc_id)
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            print(line)
    elif run_hazard is not None:
        print('WARN: --rh/--run-hazard are deprecated, use --run instead',
              file=sys.stderr)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(os.path.expanduser(run_hazard), log_level,
                log_file, exports)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            print(line)
    elif run_risk is not None:
        print('WARN: --rr/--run-risk are deprecated, use --run instead',
              file=sys.stderr)
        if hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        run_job(
            os.path.expanduser(run_risk),
            log_level, log_file, exports,
            hazard_calculation_id=hc_id)

    # export
    elif make_html_report:
        print('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            print(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            print(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports or 'xml,csv'):
            print(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports or 'xml,csv'):
            print(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()
Example #13
0
def main():
    arg_parser = set_up_arg_parser()

    args = arg_parser.parse_args()

    exports = args.exports or 'xml,csv'

    if args.version:
        print openquake.engine.__version__
        sys.exit(0)

    if args.run or args.run_hazard or args.run_risk:
        # the logging will be configured in engine.py
        pass
    else:
        # configure a basic logging
        logging.basicConfig(level=logging.INFO)

    if args.config_file:
        os.environ[config.OQ_CONFIG_FILE_VAR] = \
            abspath(expanduser(args.config_file))
        config.refresh()

    if args.no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the DbServer is up
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        err = sock.connect_ex(config.DBS_ADDRESS)
    finally:
        sock.close()
    if err:
        multi_user = valid.boolean(config.get('dbserver', 'multi_user'))
        if multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically
        dblog = os.path.expanduser('~/oq-dbserver.log')
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'], stderr=open(dblog, 'w'))
    if args.upgrade_db:
        logs.set_level('info')
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif args.yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if args.version_db:
        print logs.dbcmd('version_db')
        sys.exit(0)

    if args.what_if_I_upgrade:
        print logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts')
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if args.hazard_calculation_id:
        hc_id = get_job_id(args.hazard_calculation_id)
    else:
        hc_id = None
    if args.run:
        job_inis = map(expanduser, args.run.split(','))
        if len(job_inis) not in (1, 2):
            sys.exit('%s should be a .ini filename or a pair of filenames '
                     'separated by a comma' % args.run)
        for job_ini in job_inis:
            open(job_ini).read()  # raise an IOError if the file does not exist
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None

        if len(job_inis) == 2:
            # run hazard
            job_id = run_job(job_inis[0], args.log_level,
                             log_file, args.exports)
            # run risk
            run_job(job_inis[1], args.log_level, log_file,
                    args.exports, hazard_calculation_id=job_id)
        else:
            run_job(
                expanduser(args.run), args.log_level, log_file,
                args.exports, hazard_calculation_id=hc_id)
    # hazard
    elif args.list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            print line
    elif args.run_hazard is not None:
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(expanduser(args.run_hazard), args.log_level,
                log_file, args.exports)
    elif args.delete_calculation is not None:
        delete_calculation(args.delete_calculation, args.yes)
    # risk
    elif args.list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            print line
    elif args.run_risk is not None:
        if args.hazard_calculation_id is None:
            sys.exit(MISSING_HAZARD_MSG)
        log_file = expanduser(args.log_file) \
            if args.log_file is not None else None
        run_job(
            expanduser(args.run_risk),
            args.log_level, log_file, args.exports,
            hazard_calculation_id=hc_id)

    # export
    elif args.make_html_report:
        print 'Written', make_report(args.make_html_report)
        sys.exit(0)

    elif args.list_outputs is not None:
        hc_id = get_job_id(args.list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            print line
    elif args.show_view is not None:
        job_id, view_name = args.show_view
        print views.view(view_name, datastore.read(int(job_id)))
    elif args.show_log is not None:
        hc_id = get_job_id(args.show_log[0])
        for line in logs.dbcmd('get_log', hc_id):
            print line

    elif args.export_output is not None:
        output_id, target_dir = args.export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, expanduser(target_dir), exports):
            print line

    elif args.export_outputs is not None:
        job_id, target_dir = args.export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, expanduser(target_dir), exports):
            print line

    elif args.delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())
    else:
        arg_parser.print_usage()
Example #14
0
def engine(log_file, no_distribute, yes, config_file, make_html_report,
           upgrade_db, db_version, what_if_I_upgrade, run,
           list_hazard_calculations, list_risk_calculations,
           delete_calculation, delete_uncompleted_calculations,
           hazard_calculation_id, list_outputs, show_log,
           export_output, export_outputs, exports='',
           log_level='info', reuse_hazard=False):
    """
    Run a calculation using the traditional command line API
    """
    if not run:
        # configure a basic logging
        logs.init()

    if config_file:
        config.read(os.path.abspath(os.path.expanduser(config_file)),
                    soft_mem_limit=int, hard_mem_limit=int, port=int,
                    multi_user=valid.boolean, multi_node=valid.boolean)

    if no_distribute:
        os.environ['OQ_DISTRIBUTE'] = 'no'

    # check if the datadir exists
    datadir = datastore.get_datadir()
    if not os.path.exists(datadir):
        os.makedirs(datadir)

    dbserver.ensure_on()
    # check if we are talking to the right server
    err = dbserver.check_foreign()
    if err:
        sys.exit(err)

    if upgrade_db:
        msg = logs.dbcmd('what_if_I_upgrade', 'read_scripts')
        if msg.startswith('Your database is already updated'):
            pass
        elif yes or confirm('Proceed? (y/n) '):
            logs.dbcmd('upgrade_db')
        sys.exit(0)

    if db_version:
        safeprint(logs.dbcmd('db_version'))
        sys.exit(0)

    if what_if_I_upgrade:
        safeprint(logs.dbcmd('what_if_I_upgrade', 'extract_upgrade_scripts'))
        sys.exit(0)

    # check if the db is outdated
    outdated = logs.dbcmd('check_outdated')
    if outdated:
        sys.exit(outdated)

    # hazard or hazard+risk
    if hazard_calculation_id == -1:
        # get the latest calculation of the current user
        hc_id = get_job_id(hazard_calculation_id, getpass.getuser())
    elif hazard_calculation_id:
        # make it possible to use calculations made by another user
        hc_id = get_job_id(hazard_calculation_id)
    else:
        hc_id = None
    if run:
        log_file = os.path.expanduser(log_file) \
            if log_file is not None else None
        job_inis = [os.path.expanduser(f) for f in run]
        if len(job_inis) == 1 and not hc_id:
            # init logs before calling get_oqparam
            logs.init('nojob', getattr(logging, log_level.upper()))
            # not using logs.handle that logs on the db
            oq = readinput.get_oqparam(job_inis[0])
            smart_run(job_inis[0], oq, log_level, log_file,
                      exports, reuse_hazard)
            return
        for i, job_ini in enumerate(job_inis):
            open(job_ini, 'rb').read()  # IOError if the file does not exist
            job_id = run_job(job_ini, log_level, log_file,
                             exports, hazard_calculation_id=hc_id)
            if not hc_id:  # use the first calculation as base for the others
                hc_id = job_id
    # hazard
    elif list_hazard_calculations:
        for line in logs.dbcmd(
                'list_calculations', 'hazard', getpass.getuser()):
            safeprint(line)
    elif delete_calculation is not None:
        del_calculation(delete_calculation, yes)
    # risk
    elif list_risk_calculations:
        for line in logs.dbcmd('list_calculations', 'risk', getpass.getuser()):
            safeprint(line)

    # export
    elif make_html_report:
        safeprint('Written %s' % make_report(make_html_report))
        sys.exit(0)

    elif list_outputs is not None:
        hc_id = get_job_id(list_outputs)
        for line in logs.dbcmd('list_outputs', hc_id):
            safeprint(line)
    elif show_log is not None:
        hc_id = get_job_id(show_log)
        for line in logs.dbcmd('get_log', hc_id):
            safeprint(line)

    elif export_output is not None:
        output_id, target_dir = export_output
        dskey, calc_id, datadir = logs.dbcmd('get_output', int(output_id))
        for line in core.export_output(
                dskey, calc_id, datadir, os.path.expanduser(target_dir),
                exports or 'csv,xml'):
            safeprint(line)

    elif export_outputs is not None:
        job_id, target_dir = export_outputs
        hc_id = get_job_id(job_id)
        for line in core.export_outputs(
                hc_id, os.path.expanduser(target_dir), exports or 'csv,xml'):
            safeprint(line)

    elif delete_uncompleted_calculations:
        logs.dbcmd('delete_uncompleted_calculations', getpass.getuser())

    else:
        engine.parentparser.prog = 'oq engine'
        engine.parentparser.print_usage()