def parse_packages(self):
discovery = PluginDiscoveryService(self.project)
model_plugins = discovery.get_plugins_of_type(PluginType.MODELS)
for package in self.packages():
if not package.topics and package.tables:
raise MeltanoAnalysisMissingTopicFilesError(
f"Missing topic file(s) for package {package}")
for table in package.tables:
conf = self.parse_m5o_file(table)
parsed_table = self.table(conf, table.name)
self.tables.append(parsed_table)
for topic in package.topics:
conf = self.parse_m5o_file(topic)
parsed_topic = self.topic(conf, topic.name)
model = next(
(plugin for plugin in model_plugins
if plugin.name == package.name),
None,
)
parsed_topic[
"plugin_namespace"] = model.namespace if model else None
parsed_topic["namespace"] = package.name
self.packaged_topics.append(parsed_topic)
# Reset the tables list so that tables with the same name from
# different packages are not interfearing with correctly parsing
# the packages that follow
self.tables = []
return self.packaged_topics
def add_related_plugins(project,
plugins,
add_service: ProjectAddService,
plugin_types=list(PluginType)):
discovery_service = PluginDiscoveryService(project)
added_plugins = []
for plugin_install in plugins:
try:
plugin_def = discovery_service.find_plugin(plugin_install.type,
plugin_install.name)
except PluginNotFoundError:
continue
related_plugins = add_service.add_related(plugin_def,
plugin_types=plugin_types)
for related_plugin in related_plugins:
if related_plugin.should_add_to_file(project):
click.secho(
f"Added related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project",
fg="green",
)
else:
click.secho(
f"Adding related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project...",
fg="green",
)
added_plugins.extend(related_plugins)
return added_plugins
def installed():
"""Returns JSON of all installed plugins
Fuses the discovery.yml data with meltano.yml data and sorts each type alphabetically by name
"""
project = Project.find()
config = ConfigService(project)
discovery = PluginDiscoveryService(project)
installed_plugins = {}
# merge definitions
for plugin in sorted(config.plugins(), key=lambda x: x.name):
try:
definition = discovery.find_plugin(plugin.type, plugin.name)
merged_plugin_definition = {
**definition.canonical(),
**plugin.canonical()
}
except PluginNotFoundError:
merged_plugin_definition = {**plugin.canonical()}
merged_plugin_definition.pop("settings", None)
merged_plugin_definition.pop("select", None)
if not plugin.type in installed_plugins:
installed_plugins[plugin.type] = []
installed_plugins[plugin.type].append(merged_plugin_definition)
return jsonify({
**project.meltano.canonical(), "plugins": installed_plugins
})
def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration profiles
"""
project = Project.find()
settings = PluginSettingsService(project, show_hidden=False)
plugin = ConfigService(project).get_plugin(plugin_ref)
discovery_service = PluginDiscoveryService(project)
try:
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
settings_group_validation = plugin_def.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify({
"profiles":
profiles,
"settings":
Canonical.as_canonical(settings.definitions(plugin)),
"settings_group_validation":
settings_group_validation,
})
def all():
project = Project.find()
discovery = PluginDiscoveryService(project)
ordered_plugins = {}
for type, plugins in groupby(discovery.plugins(), key=lambda p: p.type):
ordered_plugins[type] = [plugin.canonical() for plugin in plugins]
return jsonify(ordered_plugins)
def all():
project = Project.find()
discovery = PluginDiscoveryService(project)
all_plugins = {
plugin_type:
[plugin_def_json(plugin_def) for plugin_def in plugin_defs]
for plugin_type, plugin_defs in discovery.plugins_by_type().items()
}
return jsonify(all_plugins)
def discover(project, plugin_type):
discover_service = PluginDiscoveryService(project)
try:
discovery_dict = discover_service.discover(plugin_type)
for plugin_type, plugins in discovery_dict.items():
click.secho(plugin_type, fg="green")
for plugin in plugins:
click.echo(plugin)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_discover(plugin_type=plugin_type)
except Exception as e:
click.secho("Cannot list available plugins.", fg="red")
raise click.ClickException(str(e))
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(add_service, project, PluginType.EXTRACTORS, extractor)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
add_plugin(add_service, project, PluginType.LOADERS, loader)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
add_plugin(add_service, project, PluginType.TRANSFORMERS, "dbt")
transform_add_service = TransformAddService(project)
try:
# the extractor name should match the transform name
plugin = config_service.find_plugin(
extractor, plugin_type=PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service.update_dbt_project(plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
PluginDiscoveryService(project).find_plugin(
PluginType.TRANSFORMS, extractor)
click.secho(
f"Transform '{extractor}' is missing, trying to install it...",
fg="yellow",
)
add_transform(project, extractor)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
def __init__(
self,
project,
config_service: ConfigService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.discovery_service = plugin_discovery_service or PluginDiscoveryService(
project)
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
# We use the DiscoveryService rather than the ConfigService because the
# plugin may not actually be installed yet at this point.
discovery = PluginDiscoveryService(project)
plugin = discovery.find_plugin(plugin_type, plugin_name)
add_service = ProjectAddService(project)
related_plugins = add_service.add_related(plugin)
install_service = PluginInstallService(project)
install_service.install_plugins(related_plugins)
return jsonify([plugin.canonical() for plugin in related_plugins])
def __init__(
self,
project: Project,
mail_service: MailService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.mail_service = mail_service or MailService(project)
self.plugin_discovery_service = (plugin_discovery_service
or PluginDiscoveryService(project))
def all():
project = Project.find()
discovery = PluginDiscoveryService(project)
ordered_plugins = {}
for type, plugins in groupby(discovery.plugins(), key=lambda p: p.type):
canonical_plugins = []
for plugin in plugins:
canonical_plugin = plugin.canonical()
# let's remove all the settings related data
canonical_plugin.pop("settings", None)
canonical_plugin.pop("settings_group_validation", None)
canonical_plugins.append(canonical_plugin)
ordered_plugins[type] = canonical_plugins
return jsonify(ordered_plugins)
def discover(project, plugin_type):
discover_service = PluginDiscoveryService(project)
if plugin_type == "all":
plugin_types = list(PluginType)
else:
plugin_types = [PluginType.from_cli_argument(plugin_type)]
for i, plugin_type in enumerate(plugin_types):
if i > 0:
click.echo()
click.secho(f"{str(plugin_type).capitalize()}", fg="green")
for plugin_def in discover_service.get_plugins_of_type(plugin_type):
click.echo(plugin_def.name, nl=False)
if len(plugin_def.variants) > 1:
click.echo(f", variants: {plugin_def.variant_labels}")
else:
click.echo()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_discover(plugin_type=plugin_type)
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
def __init__(
self,
project,
loader: PluginContext,
job: Optional[Job] = None,
extractor: Optional[PluginContext] = None,
plugin_settings_service: PluginSettingsService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.job = job
self.loader = loader
self.extractor = extractor
self.plugin_settings_service = plugin_settings_service
self.plugin_discovery_service = (plugin_discovery_service
or PluginDiscoveryService(project))
def __init__(
self,
project,
session,
job: Optional[Job] = None,
extractor: Optional[PluginContext] = None,
loader: Optional[PluginContext] = None,
transformer: Optional[PluginContext] = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.job = job
self.extractor = extractor
self.loader = loader
self.transformer = transformer
self.plugin_discovery_service = (plugin_discovery_service
or PluginDiscoveryService(project))
def __init__(
self,
project,
config_service: ConfigService = None,
plugin_discovery_service: PluginDiscoveryService = None,
show_hidden=True,
env_override={},
config_override={},
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.discovery_service = plugin_discovery_service or PluginDiscoveryService(
project
)
self.show_hidden = show_hidden
self.env_override = env_override
self.config_override = config_override
self._env = None
def __init__(
self,
project: Project,
config_service: ConfigService = None,
plugin_settings_service: PluginSettingsService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.plugin_discovery_service = (
plugin_discovery_service
or PluginDiscoveryService(project, config_service=config_service))
self.plugin_settings_service = plugin_settings_service or PluginSettingsService(
project,
config_service=config_service,
plugin_discovery_service=plugin_discovery_service,
)
self._extractor = None
self._loader = None
self._job = None
def plugin_discovery_service(project, discovery, config_service):
return PluginDiscoveryService(project,
discovery=discovery,
config_service=config_service)
def add_plugin(
add_service,
project: Project,
plugin_type: PluginType,
plugin_name: str,
include_related=False,
):
try:
plugin = add_service.add(plugin_type, plugin_name)
click.secho(f"Added '{plugin_name}' to your Meltano project.",
fg="green")
except PluginAlreadyAddedException as err:
click.secho(
f"'{plugin_name}' was found in your Meltano project. Use `meltano install` to install it.",
fg="yellow",
err=True,
)
plugin = err.plugin
except (PluginNotSupportedException, PluginNotFoundError):
click.secho(f"Error: {plugin_type} '{plugin_name}' is not supported.",
fg="red")
raise click.Abort()
install_service = PluginInstallService(project)
try:
click.secho(f"Installing '{plugin_name}'...")
run = install_service.install_plugin(plugin)
if run:
click.secho(run.stdout)
click.secho(f"Installed '{plugin_name}'.", fg="green")
click.secho(f"Added and installed {plugin_type} '{plugin_name}'.",
fg="green")
except PluginNotInstallable as install_err:
logging.info(f"{plugin_type} is not installable, skipping install.")
except SubprocessError as proc_err:
click.secho(str(proc_err), fg="red")
click.secho(proc_err.process.stderr, err=True)
raise click.Abort()
docs_link = plugin._extras.get("docs")
if docs_link:
click.secho(
f"Visit {docs_link} for more details about '{plugin.name}'.")
if include_related:
discovery_service = PluginDiscoveryService(project)
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
related_plugins = add_service.add_related(plugin_def)
if len(related_plugins) == 0:
click.secho(
"No related plugins found that are not already installed.")
else:
for plugin in related_plugins:
click.secho(
f"Added related plugin '{plugin.name}' to your Meltano project.",
fg="green",
)
click.secho(
f"Installing {len(related_plugins)} related plugins...")
install_status = install_service.install_plugins(related_plugins)
num_installed = len(install_status["installed"])
num_failed = len(install_status["errors"])
fg = "green"
if num_failed >= 0 and num_installed == 0:
fg = "red"
elif num_failed > 0 and num_installed > 0:
fg = "yellow"
click.secho(
f"Installed {num_installed}/{num_installed+num_failed} related plugins.",
fg=fg,
)
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
plugins = []
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
plugin = add_plugin(project,
PluginType.EXTRACTORS,
extractor,
add_service=add_service)
plugins.append(plugin)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.LOADERS,
loader,
add_service=add_service)
plugins.append(plugin)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.TRANSFORMERS,
"dbt",
add_service=add_service)
plugins.append(plugin)
discovery_service = PluginDiscoveryService(project)
extractor_plugin_def = discovery_service.find_plugin(
PluginType.EXTRACTORS, extractor)
try:
transform_plugin = config_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service = TransformAddService(project)
transform_add_service.update_dbt_project(transform_plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
transform_plugin_def = discovery_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
click.secho(
f"Transform '{transform_plugin_def.name}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(
project,
PluginType.TRANSFORMS,
transform_plugin_def.name,
add_service=add_service,
)
plugins.append(plugin)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
return install_plugins(project, plugins)
def plugin_discovery_service(project, discovery):
return PluginDiscoveryService(project, discovery=discovery)