def sync_get_external_repositories(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle,
)
repos = []
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
external_repository_data = check.inst(
execute_unary_api_cli_command(
repository_location_handle.executable_path,
'repository',
RepositoryPythonOrigin(repository_location_handle.executable_path, pointer),
),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def sync_get_external_repositories_grpc(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle,
)
repos = []
with ephemeral_grpc_api_client(
python_executable_path=repository_location_handle.executable_path
) as api_client:
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
external_repository_data = check.inst(
api_client.external_repository(
repository_python_origin=RepositoryPythonOrigin(
repository_location_handle.executable_path, pointer
)
),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def define_test_snapshot_context():
return DagsterGraphQLOutOfProcessRepositoryContext(
instance=DagsterInstance.ephemeral(),
execution_manager=SynchronousExecutionManager(),
external_repository=ExternalRepository.from_repository_def(
define_repository()),
)
def get_external_repository_from_image(image):
check.str_param(image, 'image')
with get_temp_dir(in_directory=get_system_temp_directory()) as tmp_dir:
output_file_name = "{}.json".format(uuid4())
command = 'dagster api snapshot repository'.format(
output_file=os.path.join(DEFAULT_INTERNAL_VOLUME, output_file_name)
)
output = run_serialized_container_command(
image=image,
command=command,
volumes={tmp_dir: {'bind': DEFAULT_INTERNAL_VOLUME, 'mode': DEFAULT_MODE}},
)
if len(output) != 1:
print(output)
raise DagsterInvariantViolationError(
"Running command {command} in container {image} resulted in output of length "
"{actual} lines, expected {expected} lines".format(
command=command, image=image, actual=len(output), expected=1
)
)
serialized_external_repo_data = output[0]
external_repo_data = deserialize_json_to_dagster_namedtuple(serialized_external_repo_data)
if not isinstance(external_repo_data, ExternalRepositoryData):
raise DagsterInvariantViolationError(
"Deserialized snapshot is of type {received} must be a ExternalRepositoryData".format(
received=type(external_repo_data)
)
)
return ExternalRepository(external_repo_data)
def sync_get_external_repositories_grpc(api_client, repository_location_handle):
check.inst_param(
repository_location_handle, "repository_location_handle", RepositoryLocationHandle
)
repos = []
for repository_name in repository_location_handle.repository_names:
external_repository_data = check.inst(
api_client.external_repository(
external_repository_origin=ExternalRepositoryOrigin(
repository_location_handle.origin, repository_name,
)
),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def sync_get_streaming_external_repositories_grpc(api_client, repository_location_handle):
check.inst_param(
repository_location_handle, "repository_location_handle", RepositoryLocationHandle
)
repos = []
for repository_name in repository_location_handle.repository_names:
external_repository_chunks = list(
api_client.streaming_external_repository(
external_repository_origin=ExternalRepositoryOrigin(
repository_location_handle.origin, repository_name,
)
)
)
external_repository_data = deserialize_json_to_dagster_namedtuple(
"".join(
[
chunk["serialized_external_repository_chunk"]
for chunk in external_repository_chunks
]
)
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def test_create_app_with_external_repo_data():
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml'))
assert create_app_with_external_repository(
ExternalRepository.from_repository_def(recon_repo.get_definition()),
DagsterInstance.ephemeral(),
)
def sync_get_external_repositories_grpc(api_client,
repository_location_handle):
check.inst_param(repository_location_handle, 'repository_location_handle',
RepositoryLocationHandle)
repos = []
for repository_name in repository_location_handle.repository_names:
external_repository_data = check.inst(
api_client.external_repository(
repository_grpc_server_origin=RepositoryGrpcServerOrigin(
repository_location_handle.host,
repository_location_handle.port,
repository_location_handle.socket,
repository_name,
)),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_location_handle=repository_location_handle,
),
))
return repos
def __init__(self, recon_repo, execution_manager, reloader=None):
self._recon_repo = check.inst_param(recon_repo, 'recon_repo',
ReconstructableRepository)
external_repo = ExternalRepository.from_repository_def(
recon_repo.get_definition())
self._repositories = {external_repo.name: external_repo}
self.execution_manager = check.inst_param(
execution_manager, 'pipeline_execution_manager',
PipelineExecutionManager)
self.reloader = check.opt_inst_param(reloader, 'reloader', Reloader)
def test_container_snapshot_provider(mocker):
execute_container_mock = mocker.patch(
'dagster.core.definitions.container.run_serialized_container_command',
return_value=[mock_external_repository_data()],
)
external_repository = get_external_repository_from_image("foo:latest")
execute_container_mock.assert_called_with(
image="foo:latest",
command='dagster api snapshot repository',
volumes=mocker.ANY,
)
assert (external_repository.external_repository_data ==
ExternalRepository.from_repository_def(
noop_repo()).external_repository_data)
def __init__(self,
recon_repo,
execution_manager,
instance,
reloader=None,
version=None):
super(DagsterGraphQLInProcessRepositoryContext, self).__init__(
external_repository=ExternalRepository.from_repository_def(
recon_repo.get_definition()),
instance=instance,
)
self._recon_repo = check.inst_param(recon_repo, 'recon_repo',
ReconstructableRepository)
self.reloader = check.opt_inst_param(reloader, 'reloader', Reloader)
self.execution_manager = check.inst_param(
execution_manager, 'pipeline_execution_manager',
PipelineExecutionManager)
self.version = version
def check_repo_and_scheduler(repository: ExternalRepository,
instance: DagsterInstance) -> None:
check.inst_param(repository, "repository", ExternalRepository)
check.inst_param(instance, "instance", DagsterInstance)
repository_name = repository.name
if not repository.get_external_schedules():
raise click.UsageError(
"There are no schedules defined for repository {name}.".format(
name=repository_name))
if not os.getenv("DAGSTER_HOME"):
raise click.UsageError((
"The environment variable $DAGSTER_HOME is not set. Dagster requires this "
"environment variable to be set to an existing directory in your filesystem "
"that contains your dagster instance configuration file (dagster.yaml).\n"
"You can resolve this error by exporting the environment variable."
"For example, you can run the following command in your shell or "
"include it in your shell configuration file:\n"
'\texport DAGSTER_HOME="~/dagster_home"'
"\n\n"))
def sync_get_external_repositories(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', RepositoryLocationHandle,
)
check.param_invariant(
isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle),
'repository_location_handle',
)
repos = []
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
with get_temp_file_name() as output_file:
parts = [
repository_location_handle.executable_path,
'-m',
'dagster',
'api',
'snapshot',
'repository',
output_file,
] + xplat_shlex_split(pointer.get_cli_args())
execute_command_in_subprocess(parts)
external_repository_data = read_unary_response(output_file)
check.inst(external_repository_data, ExternalRepositoryData)
repository_handle = RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
)
repos.append(ExternalRepository(external_repository_data, repository_handle))
return repos
def sync_get_external_repository(location_handle):
check.inst_param(location_handle, 'location_handle', LocationHandle)
with get_temp_file_name() as output_file:
parts = ['dagster', 'api', 'snapshot', 'repository', output_file] + xplat_shlex_split(
location_handle.pointer.get_cli_args()
)
returncode = subprocess.check_call(parts)
check.invariant(returncode == 0, 'dagster api cli invocation did not complete successfully')
messages = list(ipc_read_event_stream(output_file))
check.invariant(len(messages) == 1)
external_repository_data = messages[0]
check.inst(external_repository_data, ExternalRepositoryData)
return ExternalRepository(
external_repository_data,
RepositoryHandle(external_repository_data.name, location_handle),
)
def test_out_of_process_reload_location(self, graphql_context):
result = execute_dagster_graphql(graphql_context,
RELOAD_REPOSITORY_LOCATION_QUERY,
{"repositoryLocationName": "test"})
assert result
assert result.data
assert result.data["reloadRepositoryLocation"]
assert result.data["reloadRepositoryLocation"][
"__typename"] == "RepositoryLocation"
assert result.data["reloadRepositoryLocation"]["name"] == "test"
assert result.data["reloadRepositoryLocation"]["repositories"] == [{
"name":
"test_repo"
}]
assert result.data["reloadRepositoryLocation"][
"isReloadSupported"] is True
with mock.patch(
# note it where the function is *used* that needs to mocked, not
# where it is defined.
# see https://docs.python.org/3/library/unittest.mock.html#where-to-patch
"dagster.core.host_representation.handle.sync_list_repositories_grpc"
) as cli_command_mock:
with mock.patch(
# note it where the function is *used* that needs to mocked, not
# where it is defined.
# see https://docs.python.org/3/library/unittest.mock.html#where-to-patch
"dagster.core.host_representation.repository_location.sync_get_streaming_external_repositories_grpc"
) as external_repository_mock:
@repository
def new_repo():
return []
new_repo_data = external_repository_data_from_def(new_repo)
external_repository_mock.return_value = [
ExternalRepository(
new_repo_data,
RepositoryHandle(
"new_repo", graphql_context.
repository_locations[0].location_handle),
)
]
cli_command_mock.return_value = ListRepositoriesResponse(
repository_symbols=[],
executable_path=sys.executable,
repository_code_pointer_dict={
"new_repo":
CodePointer.from_python_file(__file__, "new_repo",
None)
},
)
result = execute_dagster_graphql(
graphql_context,
RELOAD_REPOSITORY_LOCATION_QUERY,
{"repositoryLocationName": "test"},
)
assert cli_command_mock.call_count == 1
assert external_repository_mock.call_count == 1
assert result.data["reloadRepositoryLocation"][
"repositories"] == [{
"name": "new_repo"
}]
def external_repo_from_def(repository_def, repository_handle):
return ExternalRepository(external_repository_data_from_def(repository_def), repository_handle)