Пример #1
0
    def test_supervised(self):
        # Due to the way the executor/supervisor process/forking logic is
        # defined, we can't really test the supervisor part of the workflow;
        # we can only test the job executor.
        mm = self.mm

        with mm:
            mm['get_job'].return_value = self.job
            mm['get_calc'].return_value = self.calc_class

            mm['fork'].return_value = 0

            engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard',
                            supervised=True)

        # Check the intermediate function calls and the flow of data:
        self.assertEqual(1, mm['get_calc'].call_count)
        self.assertEqual((('hazard', 'classical'), {}),
                         mm['get_calc'].call_args)

        self.assertEqual(1, mm['job_stats'].call_count)
        self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)

        self.assertEqual(1, mm['job_exec'].call_count)
        self.assertEqual(
            ((self.job, 'debug', ['geojson'], 'hazard',
              self.calc_instance),
             {}),
            mm['job_exec'].call_args
        )
Пример #2
0
    def test_unsupervised(self):
        mm = self.mm

        with mm:
            mm['get_job'].return_value = self.job
            mm['get_calc'].return_value = self.calc_class

            engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard',
                            supervised=False)

        # Check the intermediate function calls and the flow of data:
        self.assertEqual(1, mm['get_calc'].call_count)
        self.assertEqual((('hazard', 'classical'), {}),
                         mm['get_calc'].call_args)

        self.assertEqual(1, mm['job_stats'].call_count)
        self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)

        self.assertEqual(1, mm['job_exec'].call_count)
        self.assertEqual(
            ((self.job, 'debug', ['geojson'], 'hazard', self.calc_instance),
             {}),
            mm['job_exec'].call_args
        )

        self.assertEqual(1, mm['cleanup'].call_count)
        self.assertEqual(((1984, ), {}), mm['cleanup'].call_args)

        self.assertEqual(1, mm['get_job'].call_count)
        self.assertEqual(((1984, ), {}), mm['get_job'].call_args)
Пример #3
0
    def test_unsupervised(self):
        mm = self.mm

        with mm:
            mm['get_calc'].return_value = self.calc_class

            engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard')

        self.assertEqual(1, mm['save_job_stats'].call_count)

        # Check the intermediate function calls and the flow of data:
        self.assertEqual(1, mm['get_calc'].call_count)
        self.assertEqual((('hazard', 'classical'), {}),
                         mm['get_calc'].call_args)

        self.assertEqual(1, mm['job_stats'].call_count)
        self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)

        self.assertEqual(1, mm['job_exec'].call_count)
        self.assertEqual(
            ((self.job, 'debug', ['geojson'], 'hazard', self.calc_instance),
             {}),
            mm['job_exec'].call_args
        )

        self.assertEqual(1, mm['cleanup'].call_count)
        self.assertEqual(((self.job, ), {'terminate': engine.TERMINATE}),
                         mm['cleanup'].call_args)
Пример #4
0
def run_job(cfg, exports=None, hazard_calculation_id=None,
            hazard_output_id=None, **params):
    """
    Given the path to a job config file and a hazard_calculation_id
    or a output, run the job.
    """
    if exports is None:
        exports = []

    job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
                  hazard_output_id=hazard_output_id)
    job.is_running = True
    job.save()

    logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
    job_type = 'risk' if (
        hazard_calculation_id or hazard_output_id) else 'hazard'

    # update calculation parameters
    for name, value in params.iteritems():
        setattr(job.calculation, name, value)
    job.calculation.save()

    engine.run_calc(job, 'error', logfile, exports, job_type)
    return job
Пример #5
0
    def test_supervised(self):
        # Due to the way the executor/supervisor process/forking logic is
        # defined, we can't really test the supervisor part of the workflow;
        # we can only test the job executor.
        mm = self.mm

        with mm:
            mm['get_job'].return_value = self.job
            mm['get_calc'].return_value = self.calc_class

            mm['fork'].return_value = 0

            engine.run_calc(self.job, 'debug', 'oq.log', ['geojson'], 'hazard',
                            supervised=True)

        # Check the intermediate function calls and the flow of data:
        self.assertEqual(1, mm['get_calc'].call_count)
        self.assertEqual((('hazard', 'classical'), {}),
                         mm['get_calc'].call_args)

        self.assertEqual(1, mm['job_stats'].call_count)
        self.assertEqual(((self.job, ), {}), mm['job_stats'].call_args)

        self.assertEqual(1, mm['job_exec'].call_count)
        self.assertEqual(
            ((self.job, 'debug', ['geojson'], 'hazard',
              self.calc_instance),
             {}),
            mm['job_exec'].call_args
        )
Пример #6
0
def run_job(job_ini,
            log_level='info',
            log_file=None,
            exports='',
            username=getpass.getuser(),
            **kw):
    """
    Run a job using the specified config file and other options.

    :param str job_ini:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param username:
        Name of the user running the job
    :param kw:
        Extra parameters like hazard_calculation_id and calculation_mode
    """
    job_id = logs.init('job', getattr(logging, log_level.upper()))
    with logs.handle(job_id, log_level, log_file):
        job_ini = os.path.abspath(job_ini)
        oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
        kw['username'] = username
        eng.run_calc(job_id, oqparam, exports, **kw)
        for line in logs.dbcmd('list_outputs', job_id, False):
            safeprint(line)
    return job_id
Пример #7
0
def run_job(job_ini, log_level='info', log_file=None, exports='',
            username=getpass.getuser(), **kw):
    """
    Run a job using the specified config file and other options.

    :param str job_ini:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param username:
        Name of the user running the job
    :param kw:
        Extra parameters like hazard_calculation_id and calculation_mode
    """
    job_id = logs.init('job', getattr(logging, log_level.upper()))
    with logs.handle(job_id, log_level, log_file):
        job_ini = os.path.abspath(job_ini)
        oqparam = eng.job_from_file(job_ini, job_id, username, **kw)
        kw['username'] = username
        eng.run_calc(job_id, oqparam, exports, **kw)
        for line in logs.dbcmd('list_outputs', job_id, False):
            safeprint(line)
    return job_id
Пример #8
0
def run_job(cfg_file, log_level='info', log_file=None, exports='',
            hazard_calculation_id=None,  username=getpass.getuser(), **kw):
    """
    Run a job using the specified config file and other options.

    :param str cfg_file:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param hazard_calculation_id:
        ID of the previous calculation or None
    :param username:
        Name of the user running the job
    """
    # if the master dies, automatically kill the workers
    job_ini = os.path.abspath(cfg_file)
    job_id, oqparam = eng.job_from_file(
        job_ini, username, hazard_calculation_id)
    kw['username'] = username
    eng.run_calc(job_id, oqparam, log_level, log_file, exports,
                 hazard_calculation_id=hazard_calculation_id, **kw)
    for line in logs.dbcmd('list_outputs', job_id, False):
        safeprint(line)
    return job_id
Пример #9
0
 def test(self):
     cfg = helpers.get_data_path('event_based_hazard/job.ini')
     job_id, oq = actions.job_from_file(cfg, 'test_user')
     with tempfile.NamedTemporaryFile() as temp:
         with self.assertRaises(ZeroDivisionError), mock.patch(
                 'openquake.engine.engine._do_run_calc', lambda *args: 1/0):
             engine.run_calc(job_id, oq, 'info', temp.name, exports=[])
         logged = open(temp.name).read()
         # make sure the real error has been logged
         self.assertIn('integer division or modulo by zero', logged)
Пример #10
0
 def test(self):
     cfg = helpers.get_data_path('event_based_hazard/job.ini')
     job_id, oq = actions.job_from_file(cfg, 'test_user')
     with tempfile.NamedTemporaryFile() as temp:
         with self.assertRaises(ZeroDivisionError), mock.patch(
                 'openquake.engine.engine._do_run_calc',
                 lambda *args: 1 / 0):
             engine.run_calc(job_id, oq, 'info', temp.name, exports=[])
         logged = open(temp.name).read()
         # make sure the real error has been logged
         self.assertIn('integer division or modulo by zero', logged)
Пример #11
0
def run_calc(job_type, calc_id, calc_dir,
             callback_url=None, foreign_calc_id=None,
             dbname="platform", log_file=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_type:
        'hazard' or 'risk'
    :param calc_id:
        the calculation id on the engine
    :param calc_dir:
        the directory with the input files
    :param callback_url:
        the URL to call at the end of the calculation
    :param foreign_calc_id:
        the calculation id on the platform
    :param dbname:
        the platform database name
    """
    if job_type == 'hazard':
        job = oqe_models.OqJob.objects.get(hazard_calculation=calc_id)
    else:
        job = oqe_models.OqJob.objects.get(risk_calculation=calc_id)
    update_calculation(callback_url, status="started", engine_id=calc_id)

    exports = []
    progress_handler = ProgressHandler(callback_url, job.calculation)
    logging.root.addHandler(progress_handler)

    try:
        engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports, job_type)
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        update_calculation(callback_url, status="failed", einfo=einfo)
        return
    finally:
        logging.root.removeHandler(progress_handler)

    shutil.rmtree(calc_dir)

    # If requested to, signal job completion and trigger a migration of
    # results.
    if not None in (callback_url, foreign_calc_id):
        _trigger_migration(job, callback_url, foreign_calc_id, dbname)
Пример #12
0
def run_job(cfg_file, log_level, log_file, exports='',
            hazard_calculation_id=None):
    """
    Run a job using the specified config file and other options.

    :param str cfg_file:
        Path to calculation config (INI-style) files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param hazard_calculation_id:
        ID of the previous calculation or None
    """
    # if the master dies, automatically kill the workers
    concurrent_futures_process_monkeypatch()
    job_ini = os.path.abspath(cfg_file)
    job_id, oqparam = eng.job_from_file(
        job_ini, getpass.getuser(), hazard_calculation_id)
    calc = eng.run_calc(job_id, oqparam, log_level, log_file, exports,
                        hazard_calculation_id=hazard_calculation_id)
    calc.monitor.flush()
    for line in logs.dbcmd('list_outputs', job_id, False):
        print(line)
    return job_id
Пример #13
0
def run_calc(job_id,
             oqparam,
             calc_dir,
             log_file=None,
             hazard_calculation_id=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_id:
        the job ID
    :param calc_dir:
        the directory with the input files
    :param log_file:
        the name of the log file
    :param hazard_calculation_id:
        the previous calculation, if any
    """
    try:
        calc = engine.run_calc(job_id, oqparam, DEFAULT_LOG_LEVEL, log_file,
                               '', hazard_calculation_id)
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        raise
    calc.datastore.close()
    shutil.rmtree(calc_dir)
Пример #14
0
def run_calc(
        job_id, oqparam, calc_dir, log_file=None, hazard_calculation_id=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_id:
        the job ID
    :param calc_dir:
        the directory with the input files
    :param log_file:
        the name of the log file
    :param hazard_calculation_id:
        the previous calculation, if any
    """
    try:
        calc = engine.run_calc(job_id, oqparam, DEFAULT_LOG_LEVEL, log_file,
                               '', hazard_calculation_id)
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        raise
    calc.datastore.close()
    shutil.rmtree(calc_dir)
Пример #15
0
    def test(self):
        cfg = helpers.get_data_path('event_based_hazard/job.ini')
        job = engine.job_from_file(cfg, 'test_user')
        with tempfile.NamedTemporaryFile() as temp:
            with self.assertRaises(ZeroDivisionError), mock.patch(
                    'openquake.engine.engine._do_run_calc', lambda *args: 1/0
            ), mock.patch('openquake.engine.engine.cleanup_after_job',
                          lambda job: None):
                engine.run_calc(job, 'info', temp.name, exports=[])
            logged = open(temp.name).read()

            # make sure the real error has been logged
            self.assertIn('integer division or modulo by zero', logged)

            # also check the spurious cleanup error
            self.assertIn('TypeError: <lambda>() got an unexpected keyword '
                          "argument 'terminate'",  logged)
Пример #16
0
def run_calc(job_id,
             calc_dir,
             callback_url=None,
             foreign_calc_id=None,
             dbname="platform",
             log_file=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_id:
        the ID of the job on the engine
    :param calc_dir:
        the directory with the input files
    :param callback_url:
        the URL to call at the end of the calculation
    :param foreign_calc_id:
        the calculation id on the platform
    :param dbname:
        the platform database name
    """
    job = oqe_models.OqJob.objects.get(pk=job_id)
    update_calculation(callback_url, status="started", engine_id=job_id)

    progress_handler = ProgressHandler(callback_url, job)
    logging.root.addHandler(progress_handler)
    try:
        engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, 'xml,geojson,csv')
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        update_calculation(callback_url, status="failed", einfo=einfo)
        raise
    finally:
        logging.root.removeHandler(progress_handler)
        shutil.rmtree(calc_dir)

    # If requested to, signal job completion and trigger a migration of
    # results.
    if not None in (callback_url, foreign_calc_id):
        _trigger_migration(job, callback_url, foreign_calc_id, dbname)
Пример #17
0
def run_job(cfg, exports=None, hazard_calculation_id=None,
            hazard_output_id=None, **params):
    """
    Given the path to a job config file and a hazard_calculation_id
    or a output, run the job.
    """
    if exports is None:
        exports = []

    job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
                  hazard_output_id=hazard_output_id, **params)
    job.is_running = True
    job.save()

    logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')

    engine.run_calc(job, 'error', logfile, exports, job.job_type)
    return job
Пример #18
0
def run_risk_calc(calc_id, migration_callback_url=None, owner_user=None,
                  results_url=None):
    """
    Run a risk calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute.
    """
    job = oqe_models.OqJob.objects.get(risk_calculation=calc_id)
    exports = []
    # TODO: Log to file somewhere. But where?
    log_file = None
    # NOTE: Supervision MUST be turned off, or else the celeryd cluster
    # handling this task will leak processes!!!
    engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports, 'risk',
                    supervised=False)

    # If requested to, signal job completion and trigger a migration of
    # results.
    if not None in (migration_callback_url, owner_user, results_url):
        _trigger_migration(migration_callback_url, owner_user, results_url)
Пример #19
0
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None, **params):
    """
    Given the path to a job config file and a hazard_calculation_id
    or a output, run the job.

    :returns: a calculator object
    """
    job_id, oqparam = actions.job_from_file(
        cfg, 'openquake', 'error', [], hazard_calculation_id, **params)
    logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
    return engine.run_calc(job_id, oqparam, 'error', logfile, exports)
Пример #20
0
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None, **params):
    """
    Given the path to a job config file and a hazard_calculation_id
    or a output, run the job.

    :returns: a calculator object
    """
    job_id, oqparam = engine.job_from_file(cfg, 'openquake', 'error', [],
                                           hazard_calculation_id, **params)
    logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')
    return engine.run_calc(job_id, oqparam, 'error', logfile, exports)
Пример #21
0
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None,
            hazard_output_id=None, **params):
    """
    Given the path to a job config file and a hazard_calculation_id
    or a output, run the job.

    :returns: a calculator object
    """
    job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
                  hazard_output_id=hazard_output_id, **params)
    job.is_running = True
    job.save()

    logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')

    return engine.run_calc(job, 'error', logfile, exports)
Пример #22
0
def run_job(cfg, exports='xml,csv', hazard_calculation_id=None,
            hazard_output_id=None, **params):
    """
    Given the path to a job config file and a hazard_calculation_id
    or a output, run the job.

    :returns: a calculator object
    """
    job = get_job(cfg, hazard_calculation_id=hazard_calculation_id,
                  hazard_output_id=hazard_output_id, **params)
    job.is_running = True
    job.save()

    logfile = os.path.join(tempfile.gettempdir(), 'qatest.log')

    return engine.run_calc(job, 'error', logfile, exports)
Пример #23
0
def run_calc(job_id, calc_dir,
             callback_url=None, foreign_calc_id=None,
             dbname="platform", log_file=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_id:
        the ID of the job on the engine
    :param calc_dir:
        the directory with the input files
    :param callback_url:
        the URL to call at the end of the calculation
    :param foreign_calc_id:
        the calculation id on the platform
    :param dbname:
        the platform database name
    """
    job = oqe_models.OqJob.objects.get(pk=job_id)
    update_calculation(callback_url, status="started", engine_id=job_id)

    progress_handler = ProgressHandler(callback_url, job)
    logging.root.addHandler(progress_handler)
    try:
        calc = engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports='')
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        update_calculation(callback_url, status="failed", einfo=einfo)
        raise
    finally:
        logging.root.removeHandler(progress_handler)
    if hasattr(calc, 'datastore'):
        calc.datastore.close()
    shutil.rmtree(calc_dir)
Пример #24
0
def run_jobs(job_inis, log_level='info', log_file=None, exports='',
             username=getpass.getuser(), **kw):
    """
    Run jobs using the specified config file and other options.

    :param str job_inis:
        A list of paths to .ini files.
    :param str log_level:
        'debug', 'info', 'warn', 'error', or 'critical'
    :param str log_file:
        Path to log file.
    :param exports:
        A comma-separated string of export types requested by the user.
    :param username:
        Name of the user running the job
    :param kw:
        Extra parameters like hazard_calculation_id and calculation_mode
    """
    dist = parallel.oq_distribute()
    jobparams = []
    for job_ini in job_inis:
        # NB: the logs must be initialized BEFORE everything
        job_id = logs.init('job', getattr(logging, log_level.upper()))
        with logs.handle(job_id, log_level, log_file):
            oqparam = eng.job_from_file(os.path.abspath(job_ini), job_id,
                                        username, **kw)
        if (not jobparams and 'csm_cache' not in kw
                and 'hazard_calculation_id' not in kw):
            kw['hazard_calculation_id'] = job_id
        jobparams.append((job_id, oqparam))
    jobarray = len(jobparams) > 1 and 'csm_cache' in kw
    try:
        eng.poll_queue(job_id, poll_time=15)
        # wait for an empty slot or a CTRL-C
    except BaseException:
        # the job aborted even before starting
        for job_id, oqparam in jobparams:
            logs.dbcmd('finish', job_id, 'aborted')
        return jobparams
    else:
        for job_id, oqparam in jobparams:
            dic = {'status': 'executing', 'pid': eng._PID}
            if jobarray:
                dic['hazard_calculation_id'] = jobparams[0][0]
            logs.dbcmd('update_job', job_id, dic)
    try:
        if dist == 'zmq' and config.zworkers['host_cores']:
            logging.info('Asking the DbServer to start the workers')
            logs.dbcmd('zmq_start')  # start the zworkers
            logs.dbcmd('zmq_wait')  # wait for them to go up
        allargs = [(job_id, oqparam, exports, log_level, log_file)
                   for job_id, oqparam in jobparams]
        if jobarray:
            with start_many(eng.run_calc, allargs):
                pass
        else:
            for args in allargs:
                eng.run_calc(*args)
    finally:
        if dist == 'zmq' and config.zworkers['host_cores']:
            logging.info('Stopping the zworkers')
            logs.dbcmd('zmq_stop')
        elif dist.startswith('celery'):
            eng.celery_cleanup(config.distribution.terminate_workers_on_revoke)
    return jobparams