Python is_module_available Examples

Example #1

File: tasks.py Project: aaronsql2019/dagster

def make_app(config=None):
    config = check.opt_inst_param(config, 'config', (CeleryConfig, CeleryK8sJobConfig))

    app_args = config.app_args() if config is not None else {}

    app_ = Celery('dagster', **app_args)

    if config is None:
        app_.config_from_object('dagster_celery.defaults', force=True)

        if is_module_available('dagster_celery_config'):
            # pylint: disable=protected-access
            obj = force_mapping(app_.loader._smart_import('dagster_celery_config'))
            app_.conf.update(obj)

    app_.loader.import_module('celery.contrib.testing.tasks')

    app_.conf.task_queues = [
        Queue('dagster', routing_key='dagster.#', queue_arguments={'x-max-priority': 10})
    ]
    app_.conf.task_routes = {
        'execute_plan': {'queue': 'dagster', 'routing_key': 'dagster.execute_plan'},
        'execute_step_k8s_job': {'queue': 'dagster', 'routing_key': 'dagster.execute_step_k8s_job'},
    }
    app_.conf.task_queue_max_priority = 10
    app_.conf.task_default_priority = 5
    return app_

Example #2

File: inference.py Project: sd2k/dagster

def _infer_input_description_from_docstring(fn):
    if not is_module_available("docstring_parser"):
        return {}

    from docstring_parser import parse

    docstring = parse(fn.__doc__)
    return {p.arg_name: p.description for p in docstring.params}

Example #3

def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]:
    if not is_module_available("docstring_parser"):
        return None
    from docstring_parser import parse

    docstring = parse(fn.__doc__)
    if docstring.returns is None:
        return None

    return docstring.returns.description

Example #4

File: load_snapshot.py Project: uranusbeam/bit-dagster

def get_repository_snapshot_from_cli_args(cli_args):
    if cli_args.get('image'):
        _cli_load_invariant(
            is_module_available('docker'),
            msg='--image is not supported without dagster[docker] or the Python package docker installed.',
        )
        return get_container_snapshot(cli_args.get('image'))
    return RepositorySnapshot.from_repository_definition(
        handle_for_repo_cli_args(cli_args).build_repository_definition()
    )

Example #5

File: load_snapshot.py Project: aaronsql2019/dagster

def get_pipeline_snapshot_from_cli_args(cli_args):
    _cli_load_invariant(cli_args.get('pipeline_name') is not None)

    if cli_args.get('image'):
        _cli_load_invariant(
            is_module_available('docker'),
            msg='--image is not supported without dagster[docker] or the Python package docker installed.',
        )
        active_repo_data = get_active_repository_data_from_image(cli_args.get('image'))
        return active_repo_data.get_pipeline_snapshot(cli_args.get('pipeline_name')[0])
    else:
        pipeline_definition = handle_for_pipeline_cli_args(cli_args).build_pipeline_definition()
        return PipelineSnapshot.from_pipeline_def(pipeline_definition)

Example #6

def _infer_input_description_from_docstring(
        fn: Callable) -> Dict[str, Optional[str]]:
    doc_str = fn.__doc__
    if not is_module_available("docstring_parser") or doc_str is None:
        return {}

    from docstring_parser import parse

    try:
        docstring = parse(doc_str)
        return {p.arg_name: p.description for p in docstring.params}
    except Exception:
        return {}

Example #7

def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]:
    doc_str = fn.__doc__
    if not is_module_available("docstring_parser") or doc_str is None:
        return None
    from docstring_parser import parse

    try:
        docstring = parse(doc_str)
        if docstring.returns is None:
            return None

        return docstring.returns.description
    except Exception:
        return None

Example #8

def get_pipeline_snapshot_from_cli_args(cli_args):
    _cli_load_invariant(cli_args.get('pipeline_name') is not None)

    if cli_args.get('image'):
        _cli_load_invariant(
            is_module_available('docker'),
            msg=
            '--image is not supported without dagster[docker] or the Python package docker installed.',
        )
        external_repo = get_external_repository_from_image(
            cli_args.get('image'))
        return external_repo.get_external_pipeline(
            cli_args.get('pipeline_name')[0]).pipeline_snapshot
    else:
        pipeline_def = recon_pipeline_for_cli_args(cli_args).get_definition()
        return PipelineSnapshot.from_pipeline_def(pipeline_def)

Example #9

File: make_app.py Project: varokas/dagster-1

def make_app_with_task_routes(task_routes, app_args=None):
    app_ = Celery('dagster', **(app_args if app_args else {}))

    if app_args is None:
        app_.config_from_object('dagster_celery.defaults', force=True)

        if is_module_available('dagster_celery_config'):
            # pylint: disable=protected-access
            obj = force_mapping(app_.loader._smart_import('dagster_celery_config'))
            app_.conf.update(obj)

    app_.loader.import_module('celery.contrib.testing.tasks')

    app_.conf.task_queues = [
        Queue('dagster', routing_key='dagster.#', queue_arguments={'x-max-priority': 10})
    ]
    app_.conf.task_routes = task_routes
    app_.conf.task_queue_max_priority = 10
    app_.conf.task_default_priority = 5
    return app_