def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration
"""
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
try:
settings_group_validation = plugin.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
return jsonify({
**get_config_with_metadata(settings),
"settings":
Canonical.as_canonical(settings.definitions(extras=False)),
"settings_group_validation":
settings_group_validation,
})
def invoke(project, plugin_type, dump, plugin_name, plugin_args):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
_, Session = project_engine(project)
session = Session()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name,
plugin_type=plugin_type,
invokable=True)
try:
invoker = invoker_factory(project,
plugin,
plugins_service=plugins_service)
with invoker.prepared(session):
if dump:
dump_file(invoker, dump)
exit_code = 0
else:
handle = invoker.invoke(*plugin_args)
exit_code = handle.wait()
except SubprocessError as err:
logger.error(err.stderr)
raise
finally:
session.close()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_invoke(plugin_name=plugin_name,
plugin_args=" ".join(plugin_args))
sys.exit(exit_code)
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type)
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_service.add_related(plugin)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
related_plugins.reverse()
install_service = PluginInstallService(project,
plugins_service=plugins_service)
install_status = install_service.install_plugins(
related_plugins, reason=PluginInstallReason.ADD)
for error in install_status["errors"]:
raise PluginInstallError(error["message"])
return jsonify([plugin.canonical() for plugin in related_plugins])
def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
plugins_service = ProjectPluginsService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = plugins_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
plugins = list(plugins_service.plugins())
if include_related:
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_related_plugins(project, plugins, add_service=add_service)
plugins.extend(related_plugins)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
plugins.reverse()
click.echo(f"Installing {len(plugins)} plugins...")
success = install_plugins(project, plugins)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_install()
if not success:
raise CliError("Failed to install plugin(s)")
def save_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
config = payload.get("config", {})
for name, value in config.items():
if not validate_plugin_config(plugin, name, value, project, settings):
continue
if value == "":
settings.unset(name, session=db.session)
else:
settings.set(name, value, session=db.session)
return jsonify(get_config_with_metadata(settings))
def test_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for testing a plugin configuration's valid connection
"""
project = Project.find()
payload = request.get_json()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
config = payload.get("config", {})
valid_config = {
name: value
for name, value in config.items()
if validate_plugin_config(plugin, name, value, project, settings)
}
settings.config_override = PluginSettingsService.unredact(valid_config)
async def test_stream(tap_stream) -> bool:
while not tap_stream.at_eof():
message = await tap_stream.readline()
json_dict = json.loads(message)
if json_dict["type"] == "RECORD":
return True
return False
async def test_extractor():
process = None
try:
invoker = invoker_factory(
project,
plugin,
plugins_service=plugins_service,
plugin_settings_service=settings,
)
with invoker.prepared(db.session):
process = await invoker.invoke_async(
stdout=asyncio.subprocess.PIPE)
return await test_stream(process.stdout)
except Exception as err:
logging.debug(err)
# if anything happens, this is not successful
return False
finally:
try:
if process:
psutil.Process(process.pid).terminate()
except Exception as err:
logging.debug(err)
loop = asyncio.get_event_loop()
success = loop.run_until_complete(test_extractor())
return jsonify({"is_success": success}), 200
def config(ctx, project, plugin_type, plugin_name, format, extras):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
plugins_service = ProjectPluginsService(project)
try:
plugin = plugins_service.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
except PluginNotFoundError:
if plugin_name == "meltano":
plugin = None
else:
raise
_, Session = project_engine(project)
session = Session()
try:
if plugin:
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service)
else:
settings = ProjectSettingsService(
project, config_service=plugins_service.config_service)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
process = extras is not True
config = settings.as_dict(extras=extras,
process=process,
session=session)
print(json.dumps(config, indent=2))
elif format == "env":
env = settings.as_env(extras=extras, session=session)
with tempfile.NamedTemporaryFile() as temp_dotenv:
path = temp_dotenv.name
for key, value in env.items():
dotenv.set_key(path, key, value)
dotenv_content = Path(temp_dotenv.name).read_text()
print(dotenv_content, end="")
finally:
session.close()
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type)
install_service = PluginInstallService(project,
plugins_service=plugins_service)
install_service.install_plugin(plugin, reason=PluginInstallReason.ADD)
return jsonify(plugin.canonical())
def project_plugins_service(project, config_service, plugin_discovery_service):
return ProjectPluginsService(
project,
config_service=config_service,
discovery_service=plugin_discovery_service,
use_cache=False,
)
def __init__(
self,
project,
job: Optional[Job] = None,
session=None,
extractor: Optional[PluginContext] = None,
loader: Optional[PluginContext] = None,
transform: Optional[PluginContext] = None,
transformer: Optional[PluginContext] = None,
only_transform: Optional[bool] = False,
dry_run: Optional[bool] = False,
full_refresh: Optional[bool] = False,
select_filter: Optional[list] = [],
catalog: Optional[str] = None,
state: Optional[str] = None,
plugins_service: ProjectPluginsService = None,
):
self.project = project
self.job = job
self.session = session
self.extractor = extractor
self.loader = loader
self.transform = transform
self.transformer = transformer
self.only_transform = only_transform
self.dry_run = dry_run
self.full_refresh = full_refresh
self.select_filter = select_filter
self.catalog = catalog
self.state = state
self.plugins_service = plugins_service or ProjectPluginsService(
project)
def __init__(
self,
project: Project,
plugin: ProjectPlugin,
*args,
plugins_service: ProjectPluginsService = None,
**kwargs,
):
super().__init__(project, *args, **kwargs)
self.plugin = plugin
self.plugins_service = plugins_service or ProjectPluginsService(self.project)
project_settings_service = ProjectSettingsService(
self.project, config_service=self.plugins_service.config_service
)
self.env_override = {
**project_settings_service.env,
**project_settings_service.as_env(),
**self.env_override,
**self.plugin.info_env,
}
self._inherited_settings_service = None
def __init__(
self,
project: Project,
mail_service: MailService = None,
plugins_service: ProjectPluginsService = None,
):
self.project = project
self.mail_service = mail_service or MailService(project)
self.plugins_service = plugins_service or ProjectPluginsService(
project)
def __init__(
self,
project: Project,
extractor: str,
plugins_service: ProjectPluginsService = None,
):
self.project = project
self.plugins_service = plugins_service or ProjectPluginsService(
project)
self._extractor = self.plugins_service.find_plugin(
extractor, PluginType.EXTRACTORS)
def installed():
project = Project.find()
plugins_service = ProjectPluginsService(project)
def plugin_json(plugin: ProjectPlugin):
plugin_json = {"name": plugin.name}
try:
plugin_json.update(plugin_def_json(plugin))
plugin_json["variant"] = plugin.variant
plugin_json["docs"] = plugin.docs
except PluginNotFoundError:
pass
return plugin_json
installed_plugins = {
plugin_type: [plugin_json(plugin) for plugin in plugins]
for plugin_type, plugins in plugins_service.plugins_by_type().items()
}
return jsonify(installed_plugins)
def update_files(self):
"""
Update the files managed by Meltano inside the current project.
"""
click.secho("Updating files managed by plugins...", fg="blue")
file_plugins = ProjectPluginsService(self.project).get_plugins_of_type(
PluginType.FILES)
if not file_plugins:
click.echo("Nothing to update")
return
install_plugins(self.project,
file_plugins,
reason=PluginInstallReason.UPGRADE)
def __init__(self,
project: Project,
plugins_service: ProjectPluginsService = None):
self.project = project
self.plugins_service = plugins_service or ProjectPluginsService(
project)
self._session = None
self._job = None
self._extractor = None
self._loader = None
self._transform = None
self._transformer = None
self._only_transform = False
self._dry_run = False
self._full_refresh = False
self._select_filter = None
self._catalog = None
self._state = None
def elt(
project,
extractor,
loader,
transform,
dry,
full_refresh,
select,
exclude,
catalog,
state,
dump,
job_id,
force,
):
"""
meltano elt EXTRACTOR_NAME LOADER_NAME
extractor_name: Which extractor should be used in this extraction
loader_name: Which loader should be used in this extraction
"""
select_filter = [*select, *(f"!{entity}" for entity in exclude)]
job = Job(
job_id=job_id or
f'{datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%S")}--{extractor}--{loader}'
)
_, Session = project_engine(project)
session = Session()
try:
plugins_service = ProjectPluginsService(project)
context_builder = _elt_context_builder(
project,
job,
session,
extractor,
loader,
transform,
dry_run=dry,
full_refresh=full_refresh,
select_filter=select_filter,
catalog=catalog,
state=state,
plugins_service=plugins_service,
)
if dump:
dump_file(context_builder, dump)
else:
run_async(
_run_job(project, job, session, context_builder, force=force))
finally:
session.close()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_elt(extractor=extractor,
loader=loader,
transform=transform)
def add(
ctx,
project,
plugin_type,
plugin_name,
inherit_from=None,
variant=None,
as_name=None,
**flags,
):
"""Add a plugin to your project."""
plugin_type = PluginType.from_cli_argument(plugin_type)
plugin_names = plugin_name # nargs=-1
if as_name:
# `add <type> <inherit-from> --as <name>``
# is equivalent to:
# `add <type> <name> --inherit-from <inherit-from>``
inherit_from = plugin_names[0]
plugin_names = [as_name]
plugins_service = ProjectPluginsService(project)
if flags["custom"]:
if plugin_type in (
PluginType.TRANSFORMERS,
PluginType.TRANSFORMS,
PluginType.ORCHESTRATORS,
):
raise CliError(f"--custom is not supported for {plugin_type}")
add_service = ProjectAddService(project, plugins_service=plugins_service)
plugins = [
add_plugin(
project,
plugin_type,
plugin_name,
inherit_from=inherit_from,
variant=variant,
custom=flags["custom"],
add_service=add_service,
) for plugin_name in plugin_names
]
related_plugin_types = [PluginType.FILES]
if flags["include_related"]:
related_plugin_types = list(PluginType)
related_plugins = add_related_plugins(project,
plugins,
add_service=add_service,
plugin_types=related_plugin_types)
plugins.extend(related_plugins)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
plugins.reverse()
success = install_plugins(project, plugins, reason=PluginInstallReason.ADD)
if not success:
raise CliError("Failed to install plugin(s)")
_print_plugins(plugins)