def make_app(config=None): config = check.opt_inst_param(config, 'config', (CeleryConfig, CeleryK8sJobConfig)) app_args = config.app_args() if config is not None else {} app_ = Celery('dagster', **app_args) if config is None: app_.config_from_object('dagster_celery.defaults', force=True) if is_module_available('dagster_celery_config'): # pylint: disable=protected-access obj = force_mapping(app_.loader._smart_import('dagster_celery_config')) app_.conf.update(obj) app_.loader.import_module('celery.contrib.testing.tasks') app_.conf.task_queues = [ Queue('dagster', routing_key='dagster.#', queue_arguments={'x-max-priority': 10}) ] app_.conf.task_routes = { 'execute_plan': {'queue': 'dagster', 'routing_key': 'dagster.execute_plan'}, 'execute_step_k8s_job': {'queue': 'dagster', 'routing_key': 'dagster.execute_step_k8s_job'}, } app_.conf.task_queue_max_priority = 10 app_.conf.task_default_priority = 5 return app_
def _infer_input_description_from_docstring(fn): if not is_module_available("docstring_parser"): return {} from docstring_parser import parse docstring = parse(fn.__doc__) return {p.arg_name: p.description for p in docstring.params}
def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]: if not is_module_available("docstring_parser"): return None from docstring_parser import parse docstring = parse(fn.__doc__) if docstring.returns is None: return None return docstring.returns.description
def get_repository_snapshot_from_cli_args(cli_args): if cli_args.get('image'): _cli_load_invariant( is_module_available('docker'), msg='--image is not supported without dagster[docker] or the Python package docker installed.', ) return get_container_snapshot(cli_args.get('image')) return RepositorySnapshot.from_repository_definition( handle_for_repo_cli_args(cli_args).build_repository_definition() )
def get_pipeline_snapshot_from_cli_args(cli_args): _cli_load_invariant(cli_args.get('pipeline_name') is not None) if cli_args.get('image'): _cli_load_invariant( is_module_available('docker'), msg='--image is not supported without dagster[docker] or the Python package docker installed.', ) active_repo_data = get_active_repository_data_from_image(cli_args.get('image')) return active_repo_data.get_pipeline_snapshot(cli_args.get('pipeline_name')[0]) else: pipeline_definition = handle_for_pipeline_cli_args(cli_args).build_pipeline_definition() return PipelineSnapshot.from_pipeline_def(pipeline_definition)
def _infer_input_description_from_docstring( fn: Callable) -> Dict[str, Optional[str]]: doc_str = fn.__doc__ if not is_module_available("docstring_parser") or doc_str is None: return {} from docstring_parser import parse try: docstring = parse(doc_str) return {p.arg_name: p.description for p in docstring.params} except Exception: return {}
def _infer_output_description_from_docstring(fn: Callable) -> Optional[str]: doc_str = fn.__doc__ if not is_module_available("docstring_parser") or doc_str is None: return None from docstring_parser import parse try: docstring = parse(doc_str) if docstring.returns is None: return None return docstring.returns.description except Exception: return None
def get_pipeline_snapshot_from_cli_args(cli_args): _cli_load_invariant(cli_args.get('pipeline_name') is not None) if cli_args.get('image'): _cli_load_invariant( is_module_available('docker'), msg= '--image is not supported without dagster[docker] or the Python package docker installed.', ) external_repo = get_external_repository_from_image( cli_args.get('image')) return external_repo.get_external_pipeline( cli_args.get('pipeline_name')[0]).pipeline_snapshot else: pipeline_def = recon_pipeline_for_cli_args(cli_args).get_definition() return PipelineSnapshot.from_pipeline_def(pipeline_def)
def make_app_with_task_routes(task_routes, app_args=None): app_ = Celery('dagster', **(app_args if app_args else {})) if app_args is None: app_.config_from_object('dagster_celery.defaults', force=True) if is_module_available('dagster_celery_config'): # pylint: disable=protected-access obj = force_mapping(app_.loader._smart_import('dagster_celery_config')) app_.conf.update(obj) app_.loader.import_module('celery.contrib.testing.tasks') app_.conf.task_queues = [ Queue('dagster', routing_key='dagster.#', queue_arguments={'x-max-priority': 10}) ] app_.conf.task_routes = task_routes app_.conf.task_queue_max_priority = 10 app_.conf.task_default_priority = 5 return app_