def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration profiles
"""
project = Project.find()
settings = PluginSettingsService(project, show_hidden=False)
plugin = ConfigService(project).get_plugin(plugin_ref)
discovery_service = PluginDiscoveryService(project)
try:
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
settings_group_validation = plugin_def.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify({
"profiles":
profiles,
"settings":
Canonical.as_canonical(settings.definitions(plugin)),
"settings_group_validation":
settings_group_validation,
})
def config(ctx, project, plugin_type, plugin_name, format):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
config = ConfigService(project)
plugin = config.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
_, Session = project_engine(project)
session = Session()
try:
settings = PluginSettingsService(project).build(plugin)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
config = settings.as_config(session=session)
print(json.dumps(config))
elif format == "env":
for env, value in settings.as_env(session=session).items():
print(f"{env}={value}")
finally:
session.close()
def validate_plugin_config(plugin: PluginRef, name, value, project: Project,
settings: PluginSettingsService):
setting_def = settings.find_setting(plugin, name)
# we want to prevent the edition of protected settings from the UI
if setting_def.protected:
logging.warning("Cannot set a 'protected' configuration externally.")
return False
if setting_def.kind == "file" and value and value != "":
uploads_directory = project.extract_dir(plugin.full_name)
resolved_file_path = project.root_dir(value).resolve()
if not str(resolved_file_path).startswith(
str(uploads_directory) + "/"):
logging.warning(
"Cannot set a file configuration to a path outside the project directory"
)
return False
old_value, source = settings.get_value(db.session, plugin, name)
if source in (PluginSettingValueSource.ENV,
PluginSettingValueSource.MELTANO_YML):
logging.warning(
"Cannot override a configuration set in the environment or meltano.yml."
)
return False
return True
def setup_js_context():
# setup the appUrl
appUrl = urlsplit(request.host_url)
g.jsContext = {"appUrl": appUrl.geturl()[:-1]}
if tracker.send_anonymous_usage_stats:
g.jsContext["isSendAnonymousUsageStats"] = True
g.jsContext["projectId"] = tracker.project_id
g.jsContext["version"] = meltano.__version__
# setup the airflowUrl
try:
airflow = ConfigService(project).find_plugin("airflow")
settings = PluginSettingsService(project)
airflow_port, _ = settings.get_value(db.session, airflow,
"webserver.web_server_port")
g.jsContext["airflowUrl"] = appUrl._replace(
netloc=f"{appUrl.hostname}:{airflow_port}").geturl()[:-1]
except (PluginMissingError, PluginSettingMissingError):
pass
# setup the dbtDocsUrl
g.jsContext["dbtDocsUrl"] = appUrl._replace(
path="/-/dbt/").geturl()[:-1]
def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration
"""
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
try:
settings_group_validation = plugin.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
return jsonify({
**get_config_with_metadata(settings),
"settings":
Canonical.as_canonical(settings.definitions(extras=False)),
"settings_group_validation":
settings_group_validation,
})
def after_install(
self,
installer: PluginInstallService,
plugin: ProjectPlugin,
reason: PluginInstallReason,
):
project = installer.project
plugins_service = installer.plugins_service
plugin_settings_service = PluginSettingsService(
project, plugin, plugins_service=plugins_service)
update_config = plugin_settings_service.get("_update")
paths_to_update = [
path for path, to_update in update_config.items() if to_update
]
if reason is PluginInstallReason.ADD:
print(f"Adding '{plugin.name}' files to project...")
for path in self.create_files(project, paths_to_update):
print(f"Created {path}")
elif reason is PluginInstallReason.UPGRADE:
print(f"Updating '{plugin.name}' files in project...")
updated_paths = self.update_files(project, paths_to_update)
if not updated_paths:
print("Nothing to update")
return
for path in updated_paths:
print(f"Updated {path}")
else:
print(
f"Run `meltano upgrade files` to update your project's '{plugin.name}' files."
)
def test_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for testing a plugin configuration's valid connection
"""
project = Project.find()
payload = request.get_json()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
config = payload.get("config", {})
valid_config = {
name: value
for name, value in config.items()
if validate_plugin_config(plugin, name, value, project, settings)
}
settings.config_override = PluginSettingsService.unredact(valid_config)
async def test_stream(tap_stream) -> bool:
while not tap_stream.at_eof():
message = await tap_stream.readline()
json_dict = json.loads(message)
if json_dict["type"] == "RECORD":
return True
return False
async def test_extractor():
process = None
try:
invoker = invoker_factory(
project,
plugin,
plugins_service=plugins_service,
plugin_settings_service=settings,
)
with invoker.prepared(db.session):
process = await invoker.invoke_async(
stdout=asyncio.subprocess.PIPE)
return await test_stream(process.stdout)
except Exception as err:
logging.debug(err)
# if anything happens, this is not successful
return False
finally:
try:
if process:
psutil.Process(process.pid).terminate()
except Exception as err:
logging.debug(err)
loop = asyncio.get_event_loop()
success = loop.run_until_complete(test_extractor())
return jsonify({"is_success": success}), 200
def get_plugin_configuration(plugin_ref) -> Response:
"""
endpoint for getting a plugin's configuration
"""
project = Project.find()
settings = PluginSettingsService(project)
config = flatten(settings.as_config(db.session, plugin_ref, redacted=True),
reducer="dot")
return jsonify({
# freeze the keys because they are used for lookups
"config": freeze_keys(config),
"settings": settings.get_definition(plugin_ref).settings,
})
def plugin_config(self, project):
_, Session = project_engine(project)
session = Session()
try:
plugin_settings_service = PluginSettingsService(project)
raw_config = plugin_settings_service.as_config(session, self)
finally:
session.close()
config = {}
for key, value in raw_config.items():
nest(config, key, value, maxsplit=1)
return config
def config(ctx, project, plugin_type, plugin_name, format, extras):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
plugins_service = ProjectPluginsService(project)
try:
plugin = plugins_service.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
except PluginNotFoundError:
if plugin_name == "meltano":
plugin = None
else:
raise
_, Session = project_engine(project)
session = Session()
try:
if plugin:
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service)
else:
settings = ProjectSettingsService(
project, config_service=plugins_service.config_service)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
process = extras is not True
config = settings.as_dict(extras=extras,
process=process,
session=session)
print(json.dumps(config, indent=2))
elif format == "env":
env = settings.as_env(extras=extras, session=session)
with tempfile.NamedTemporaryFile() as temp_dotenv:
path = temp_dotenv.name
for key, value in env.items():
dotenv.set_key(path, key, value)
dotenv_content = Path(temp_dotenv.name).read_text()
print(dotenv_content, end="")
finally:
session.close()
def save_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
plugin = ConfigService(project).get_plugin(plugin_ref)
# TODO iterate pipelines and save each, also set this connector's profile (reuse `pipelineInFocusIndex`?)
settings = PluginSettingsService(project, show_hidden=False)
for profile in payload:
# select the correct profile
name = profile["name"]
plugin.use_profile(plugin.get_profile(name))
for name, value in profile["config"].items():
if not validate_plugin_config(plugin, name, value, project,
settings):
continue
if value == "":
settings.unset(db.session, plugin, name)
else:
settings.set(db.session, plugin, name, value)
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify(profiles)
def config(ctx, project, plugin_name, format):
config = ConfigService(project)
plugin = config.find_plugin(plugin_name)
_, Session = project_engine(project)
session = Session()
settings = PluginSettingsService(project)
ctx.obj["settings"] = settings
ctx.obj["plugin"] = plugin
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
print(settings.as_config(session, plugin))
if format == "env":
for env, value in settings.as_env(session, plugin).items():
print(f"{env}={value}")
def save_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
config = payload.get("config", {})
for name, value in config.items():
if not validate_plugin_config(plugin, name, value, project, settings):
continue
if value == "":
settings.unset(name, session=db.session)
else:
settings.set(name, value, session=db.session)
return jsonify(get_config_with_metadata(settings))
def plugin_context(self, plugin_ref: PluginRef, env={}, config={}):
plugin = self.plugins_service.get_plugin(plugin_ref)
return PluginContext(
plugin=plugin,
settings_service=PluginSettingsService(
self.project,
plugin,
plugins_service=self.plugins_service,
env_override=env,
config_override=config,
),
session=self._session,
)
def add_plugin_configuration_profile(plugin_ref) -> Response:
"""
Endpoint for adding a configuration profile to a plugin
"""
payload = request.get_json()
project = Project.find()
config = ConfigService(project)
plugin = config.get_plugin(plugin_ref)
settings = PluginSettingsService(project)
# create the new profile for this plugin
name = payload["name"]
profile = plugin.add_profile(slugify(name), label=name)
config.update_plugin(plugin)
profile_config = settings.profile_with_config(db.session,
plugin,
profile,
redacted=True)
freeze_profile_config_keys(profile_config)
return jsonify(profile_config)
def run(self, session, dry_run=False, models=None):
# we should probably refactor this part to have an ELTContext object already
# filled with the each plugins' configuration so we don't have to query
# multiple times for the same data.
settings_service = PluginSettingsService(self.project)
try:
load = self.connection_service.load_params()
analyze = self.connection_service.analyze_params()
env = {
# inject the inferred 'schemas' from the ELTContext
"MELTANO_LOAD_SCHEMA": load["schema"],
"MELTANO_ANALYZE_SCHEMA": analyze["schema"],
"DBT_TARGET": self.connection_service.dialect,
# inject the extractor & loader configuration as ENV variables.
# that means dbt will have access to all the configuration of
# the extractor and loader
**settings_service.as_env(session, self.context.extractor.ref),
**settings_service.as_env(session, self.context.loader.ref),
}
except Exception as e:
logging.debug("Could not inject environment to dbt.")
logging.debug(
f"Could not hydrate ENV from the EltContext: {str(e)}")
raise e
# Get an asyncio event loop and use it to run the dbt commands
loop = asyncio.get_event_loop()
loop.run_until_complete(self.dbt_service.deps())
if models is not None:
models = models.replace("-", "_")
if dry_run:
loop.run_until_complete(self.dbt_service.compile(models, env=env))
else:
loop.run_until_complete(self.dbt_service.run(models, env=env))
def validate_plugin_config(plugin: PluginRef, name, value, project: Project,
settings: PluginSettingsService):
setting_def = settings.find_setting(name)
# we want to prevent the edition of protected settings from the UI
if setting_def.protected:
logging.warning("Cannot set a 'protected' setting externally.")
return False
if setting_def.kind == "file" and value and value != "":
uploads_directory = project.extract_dir(plugin.name)
resolved_file_path = project.root_dir(value).resolve()
if not str(resolved_file_path).startswith(
str(uploads_directory) + "/"):
logging.warning(
"Cannot set a file configuration to a path outside the project directory"
)
return False
old_value, metadata = settings.get_with_metadata(name, session=db.session)
if not metadata["overwritable"]:
logging.warning("Cannot overwrite this setting.")
return False
return True
def save_plugin_configuration(plugin_ref) -> Response:
"""
endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
settings = PluginSettingsService(project)
for name, value in payload.items():
# we want to prevent the edition of protected settings from the UI
if settings.find_setting(plugin_ref, name).get("protected"):
logging.warning(
"Cannot set a 'protected' configuration externally.")
continue
if value == "":
settings.unset(db.session, plugin_ref, name)
else:
settings.set(db.session, plugin_ref, name, value)
return jsonify(settings.as_config(db.session, plugin_ref, redacted=True))
def __init__(
self,
project: Project,
config_service: ConfigService = None,
plugin_settings_service: PluginSettingsService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.plugin_discovery_service = (
plugin_discovery_service
or PluginDiscoveryService(project, config_service=config_service))
self.plugin_settings_service = plugin_settings_service or PluginSettingsService(
project,
config_service=config_service,
plugin_discovery_service=plugin_discovery_service,
)
self._extractor = None
self._loader = None
self._job = None
async def test_extractor(config={}):
try:
settings_service = settings.with_config_override(
PluginSettingsService.unredact(config))
invoker = invoker_factory(
project,
plugin,
prepare_with_session=db.session,
plugin_settings_service=settings_service,
)
process = await invoker.invoke_async(stdout=asyncio.subprocess.PIPE
)
return await test_stream(process.stdout)
except Exception as err:
# if anything happens, this is not successful
return False
finally:
try:
if process:
psutil.Process(process.pid).terminate()
except Exception as err:
logging.debug(err)
def current_select(self):
plugin_settings_service = PluginSettingsService(
self.project, self.extractor, plugins_service=self.plugins_service)
return plugin_settings_service.get("_select")
def plugin_settings_service(project, config_service, plugin_discovery_service):
return PluginSettingsService(
project,
config_service=config_service,
plugin_discovery_service=plugin_discovery_service,
)
def after_install(self, project, args=[]):
_, Session = project_engine(project)
session = Session()
plugin_config_service = PluginConfigService(
self,
config_dir=project.plugin_dir(self),
run_dir=project.run_dir(self.name),
)
plugin_settings_service = PluginSettingsService(project)
airflow_cfg_path = plugin_config_service.run_dir.joinpath(
"airflow.cfg")
stub_path = plugin_config_service.config_dir.joinpath("airflow.cfg")
invoker = invoker_factory(
project,
self,
prepare_with_session=session,
plugin_config_service=plugin_config_service,
)
try:
# generate the default `airflow.cfg`
handle = invoker.invoke("--help",
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
handle.wait()
logging.debug(f"Generated default '{str(airflow_cfg_path)}'")
# move it to the config dir
shutil.move(airflow_cfg_path, stub_path)
airflow_cfg_path = stub_path
logging.debug(f"Moved to '{str(stub_path)}'")
# open the configuration and update it
# now we let's update the config to use our stubs
airflow_cfg = configparser.ConfigParser()
with airflow_cfg_path.open() as cfg:
airflow_cfg.read_file(cfg)
logging.debug(f"Loaded '{str(airflow_cfg_path)}'")
config = {}
for key, value in plugin_settings_service.as_config(session,
self).items():
nest(config, key, str(value))
for section, cfg in config.items():
airflow_cfg[section].update(cfg)
logging.debug(f"\tUpdated section [{section}] with {cfg}")
with airflow_cfg_path.open("w") as cfg:
airflow_cfg.write(cfg)
logging.debug(f"Saved '{str(airflow_cfg_path)}'")
# we've changed the configuration here, so we need to call
# prepare again on the invoker so it re-reads the configuration
# for the Airflow plugin
invoker.prepare(session)
handle = invoker.invoke(
"initdb",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
initdb = handle.wait()
if initdb:
raise SubprocessError("airflow initdb failed", handle)
logging.debug(f"Completed `airflow initdb`")
finally:
session.close()
def _factory(plugin, **kwargs):
return PluginSettingsService(
project, plugin, plugins_service=project_plugins_service, **kwargs
)
