def get_register_repo_info(cli_args, allow_none=True):
def all_none(kwargs):
for value in kwargs.values():
if value is not None:
return False
return True
scaffolding_with_repo = True
if all_none(cli_args):
if os.path.exists(os.path.join(os.getcwd(), 'repository.yml')):
cli_args['repository_yaml'] = 'repository.yml'
elif allow_none: # register_repo_info can remain None
scaffolding_with_repo = False
register_repo_info = None
if scaffolding_with_repo:
repository_target_info = load_target_info_from_cli_args(cli_args)
entrypoint = get_acceptable_entrypoint(repository_target_info)
if entrypoint:
module = entrypoint.module_name
fn_name = entrypoint.fn_name
RegisterRepoInfo = namedtuple(
'RegisterRepoInfo', 'import_statement declaration_statement')
register_repo_info = RegisterRepoInfo(
"from {module} import {fn_name}".format(module=module,
fn_name=fn_name),
"dm.register_repository({fn_name}())".format(fn_name=fn_name),
)
else:
raise click.UsageError(
"Cannot instantiate notebook with repository definition given by a function from a file"
)
return register_repo_info
def ui(host, port, watch, sync, log, log_dir, **kwargs):
repository_target_info = load_target_info_from_cli_args(kwargs)
sys.path.append(os.getcwd())
repository_container = RepositoryContainer(repository_target_info)
if log:
def create_pipeline_run(*args, **kwargs):
return LogFilePipelineRun(log_dir, *args, **kwargs)
else:
create_pipeline_run = InMemoryPipelineRun
pipeline_run_storage = PipelineRunStorage(
create_pipeline_run=create_pipeline_run)
if watch:
observer = Observer()
handler = ReloaderHandler(repository_container)
observer.schedule(handler,
os.path.dirname(os.path.abspath(os.getcwd())),
recursive=True)
observer.start()
try:
app = create_app(repository_container,
pipeline_run_storage,
use_synchronous_execution_manager=sync)
server = pywsgi.WSGIServer((host, port),
app,
handler_class=WebSocketHandler)
print('Serving on http://{host}:{port}'.format(host=host, port=port))
server.serve_forever()
except KeyboardInterrupt:
if watch:
observer.stop()
def ui(variables, query, **kwargs):
repository_target_info = load_target_info_from_cli_args(kwargs)
repository_container = RepositoryContainer(repository_target_info)
query = query.strip('\'" \n\t')
execute_query_from_cli(repository_container, query, variables)
def execute_create_notebook(notebook, solid_name, force_overwrite, **kwargs):
if not re.match(r'^[a-zA-Z0-9\-_\\/]+$', notebook):
raise click.BadOptionUsage(
notebook,
('Notebook name {name} is not valid, '
'cannot contain anything except alphanumeric characters, '
'-, _, \\ and / for path manipulation').format(name=notebook),
)
notebook_path = os.path.join(
os.getcwd(),
notebook if notebook.endswith('.ipynb') else notebook + ".ipynb")
notebook_dir = os.path.dirname(notebook_path)
if not os.path.exists(notebook_dir):
os.makedirs(notebook_dir)
if not force_overwrite and safe_isfile(notebook_path):
click.confirm(
('Warning, {notebook_path} already exists and continuing '
'will overwrite the existing notebook. '
'Are you sure you want to continue?').format(
notebook_path=notebook_path),
abort=True,
)
if not solid_name:
solid_name = os.path.basename(notebook_path).split(".")[0]
repository_target_info = load_target_info_from_cli_args(kwargs)
module_target_info = get_module_target_function(repository_target_info)
if module_target_info:
module = module_target_info.module_name
fn_name = module_target_info.fn_name
RegisterRepoInfo = namedtuple(
'RegisterRepoInfo', 'import_statement declaration_statement')
register_repo_info = RegisterRepoInfo(
"from {module} import {fn_name}".format(module=module,
fn_name=fn_name),
"dm.declare_as_solid({fn_name}(), '{solid_name}')".format(
fn_name=fn_name, solid_name=solid_name),
)
else:
raise click.UsageError(
"Cannot instantiate notebook with repository definition given by a function from a file"
)
with open(notebook_path, 'w') as f:
f.write(get_notebook_scaffolding(register_repo_info))
click.echo("Created new dagstermill notebook at {path}".format(
path=notebook_path))
def ui(host, port, watch, sync, log, log_dir, query, variables, **kwargs):
repository_target_info = load_target_info_from_cli_args(kwargs)
sys.path.append(os.getcwd())
repository_container = RepositoryContainer(repository_target_info)
if query:
return execute_query_from_cli(repository_container, query, variables)
else:
if variables:
raise Exception('if you specify --variables/-v you need to specify --query/-q')
host_dagit_ui(log, log_dir, watch, repository_container, sync, host, port)
def ui(host, port, sync, log, log_dir, no_watch=False, **kwargs):
repository_target_info = load_target_info_from_cli_args(kwargs)
# add the path for the cwd so imports in dynamically loaded code work correctly
sys.path.append(os.getcwd())
repository_container = RepositoryContainer(repository_target_info)
check.invariant(
not no_watch,
'Do not set no_watch when calling the Dagit Python CLI directly -- this flag is a no-op'
'at this level and should be set only when invoking dagit/bin/dagit.',
)
host_dagit_ui(log, log_dir, repository_container, sync, host, port)
def ui(host, port, **kwargs):
repository_target_info = load_target_info_from_cli_args(kwargs)
dynamic_obj_repo = load_repository_object_from_target_info(
repository_target_info)
sys.path.append(os.getcwd())
repository_container = RepositoryContainer(dynamic_obj_repo)
observer = Observer()
handler = ReloaderHandler(repository_container)
observer.schedule(handler,
os.path.dirname(os.path.abspath(os.getcwd())),
recursive=True)
observer.start()
try:
app = create_app(repository_container)
serve(app, host=host, port=port)
except KeyboardInterrupt:
observer.stop()