def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
config_service = ConfigService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = config_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
plugins = list(config_service.plugins())
if include_related:
add_service = ProjectAddService(project, config_service=config_service)
related_plugins = add_related_plugins(project, plugins, add_service=add_service)
plugins.extend(related_plugins)
click.echo(f"Installing {len(plugins)} plugins...")
success = install_plugins(project, plugins)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_install()
if not success:
raise click.Abort()
def save_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
plugin = ConfigService(project).get_plugin(plugin_ref)
# TODO iterate pipelines and save each, also set this connector's profile (reuse `pipelineInFocusIndex`?)
settings = PluginSettingsService(project, show_hidden=False)
for profile in payload:
# select the correct profile
name = profile["name"]
plugin.use_profile(plugin.get_profile(name))
for name, value in profile["config"].items():
if not validate_plugin_config(plugin, name, value, project,
settings):
continue
if value == "":
settings.unset(db.session, plugin, name)
else:
settings.set(db.session, plugin, name, value)
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify(profiles)
def installed():
"""Returns JSON of all installed plugins
Fuses the discovery.yml data with meltano.yml data and sorts each type alphabetically by name
"""
project = Project.find()
config = ConfigService(project)
discovery = PluginDiscoveryService(project)
installed_plugins = {}
# merge definitions
for plugin in sorted(config.plugins(), key=lambda x: x.name):
try:
definition = discovery.find_plugin(plugin.type, plugin.name)
merged_plugin_definition = {
**definition.canonical(),
**plugin.canonical()
}
except PluginNotFoundError:
merged_plugin_definition = {**plugin.canonical()}
merged_plugin_definition.pop("settings", None)
merged_plugin_definition.pop("select", None)
if not plugin.type in installed_plugins:
installed_plugins[plugin.type] = []
installed_plugins[plugin.type].append(merged_plugin_definition)
return jsonify({
**project.meltano.canonical(), "plugins": installed_plugins
})
def config(ctx, project, plugin_type, plugin_name, format):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
config = ConfigService(project)
plugin = config.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
_, Session = project_engine(project)
session = Session()
try:
settings = PluginSettingsService(project).build(plugin)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
config = settings.as_config(session=session)
print(json.dumps(config))
elif format == "env":
for env, value in settings.as_env(session=session).items():
print(f"{env}={value}")
finally:
session.close()
def invoke(project, plugin_name, plugin_args):
_, Session = project_engine(project)
try:
session = Session()
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name)
service = invoker_factory(project,
plugin,
prepare_with_session=session)
handle = service.invoke(*plugin_args)
exit_code = handle.wait()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_invoke(plugin_name=plugin_name,
plugin_args=" ".join(plugin_args))
sys.exit(exit_code)
except Exception as err:
logging.exception(err)
click.secho(f"An error occured: {err}.", fg="red")
raise click.Abort() from err
finally:
session.close()
def __init__(self, project: Project):
super().__init__(name="AirflowWorker")
self.project = project
self.config_service = ConfigService(project)
self._plugin = None
self.pid_file = PIDFile(
self.project.run_dir("airflow", "scheduler.pid"))
def test_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for testing a plugin configuration's valid connection
"""
project = Project.find()
payload = request.get_json()
config_service = ConfigService(project)
plugin = config_service.get_plugin(plugin_ref)
# load the correct profile
plugin.use_profile(plugin.get_profile(payload.get("profile")))
settings = PluginSettingsService(project, show_hidden=False)
config = payload.get("config", {})
valid_config = {
name: value
for name, value in config.items()
if validate_plugin_config(plugin, name, value, project, settings)
}
async def test_stream(tap_stream) -> bool:
while not tap_stream.at_eof():
message = await tap_stream.readline()
json_dict = json.loads(message)
if json_dict["type"] == "RECORD":
return True
return False
async def test_extractor(config={}):
try:
settings_service = settings.with_config_override(
PluginSettingsService.unredact(config))
invoker = invoker_factory(
project,
plugin,
prepare_with_session=db.session,
plugin_settings_service=settings_service,
)
process = await invoker.invoke_async(stdout=asyncio.subprocess.PIPE
)
return await test_stream(process.stdout)
except Exception as err:
# if anything happens, this is not successful
return False
finally:
try:
if process:
psutil.Process(process.pid).terminate()
except Exception as err:
logging.debug(err)
loop = asyncio.get_event_loop()
success = loop.run_until_complete(test_extractor(valid_config))
return jsonify({"is_success": success}), 200
def __init__(self, project: Project):
super().__init__(name="AirflowWorker")
self.project = project
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self._plugin = None
self._webserver = None
self._scheduler = None
def __init__(self, project: Project, loader: str, loop=None):
super().__init__()
self.project = project
self.loader = loader
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self.observer = None
self._plugin = None
self._loop = loop or asyncio.get_event_loop()
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(add_service, project, PluginType.EXTRACTORS, extractor)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
add_plugin(add_service, project, PluginType.LOADERS, loader)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
add_plugin(add_service, project, PluginType.TRANSFORMERS, "dbt")
transform_add_service = TransformAddService(project)
try:
# the extractor name should match the transform name
plugin = config_service.find_plugin(
extractor, plugin_type=PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service.update_dbt_project(plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
PluginDiscoveryService(project).find_plugin(
PluginType.TRANSFORMS, extractor)
click.secho(
f"Transform '{extractor}' is missing, trying to install it...",
fg="yellow",
)
add_transform(project, extractor)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type)
install_service = PluginInstallService(project)
install_service.install_plugin(plugin)
return jsonify(plugin.canonical())
def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration profiles
"""
project = Project.find()
settings = PluginSettingsService(project, show_hidden=False)
plugin = ConfigService(project).get_plugin(plugin_ref)
discovery_service = PluginDiscoveryService(project)
try:
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
settings_group_validation = plugin_def.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify({
"profiles":
profiles,
"settings":
Canonical.as_canonical(settings.definitions(plugin)),
"settings_group_validation":
settings_group_validation,
})
def setup_js_context():
# setup the appUrl
appUrl = urlsplit(request.host_url)
g.jsContext = {"appUrl": appUrl.geturl()[:-1]}
if tracker.send_anonymous_usage_stats:
g.jsContext["isSendAnonymousUsageStats"] = True
g.jsContext["projectId"] = tracker.project_id
g.jsContext["version"] = meltano.__version__
# setup the airflowUrl
try:
airflow = ConfigService(project).find_plugin("airflow")
settings = PluginSettingsService(project)
airflow_port, _ = settings.get_value(db.session, airflow,
"webserver.web_server_port")
g.jsContext["airflowUrl"] = appUrl._replace(
netloc=f"{appUrl.hostname}:{airflow_port}").geturl()[:-1]
except (PluginMissingError, PluginSettingMissingError):
pass
# setup the dbtDocsUrl
g.jsContext["dbtDocsUrl"] = appUrl._replace(
path="/-/dbt/").geturl()[:-1]
def __init__(self,
project: Project,
extractor: str,
config_service: ConfigService = None):
self.project = project
self.config = config_service or ConfigService(project)
self._extractor = self.config.find_plugin(extractor,
PluginType.EXTRACTORS)
def config(ctx, project, plugin_name, format):
config = ConfigService(project)
plugin = config.find_plugin(plugin_name)
_, Session = project_engine(project)
session = Session()
settings = PluginSettingsService(project)
ctx.obj["settings"] = settings
ctx.obj["plugin"] = plugin
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
print(settings.as_config(session, plugin))
if format == "env":
for env, value in settings.as_env(session, plugin).items():
print(f"{env}={value}")
def __init__(
self,
project,
config_service: ConfigService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.discovery_service = plugin_discovery_service or PluginDiscoveryService(
project)
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
compiler = ProjectCompiler(project)
install_service = PluginInstallService(project)
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type)
run_venv = install_service.create_venv(plugin)
run_install_plugin = install_service.install_plugin(plugin)
if plugin_type is PluginType.MODELS:
try:
compiler.compile()
except Exception as e:
pass
return jsonify(plugin.canonical())
def start(ctx, reload, bind_port, bind):
project = ctx.obj["project"]
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_ui()
workers = []
if not truthy(os.getenv("MELTANO_DISABLE_AIRFLOW", False)):
try:
config_service = ConfigService(project)
config_service.find_plugin("airflow")
workers.append(AirflowWorker(project))
except PluginMissingError:
pass
try:
compiler_worker = MeltanoCompilerWorker(project)
compiler_worker.compiler.compile()
workers.append(compiler_worker)
except Exception as e:
logger.error(f"Initial compilation failed: {e}")
workers.append(UIAvailableWorker("http://localhost:{bind_port}"))
workers.append(
APIWorker(
project,
f"{bind}:{bind_port}",
reload=reload or os.getenv("FLASK_ENV") == "development",
))
cleanup = start_workers(workers)
def handle_terminate(signal, frame):
cleanup()
signal.signal(signal.SIGTERM, handle_terminate)
logger.info("All workers started.")
def add_plugin_configuration_profile(plugin_ref) -> Response:
"""
Endpoint for adding a configuration profile to a plugin
"""
payload = request.get_json()
project = Project.find()
config = ConfigService(project)
plugin = config.get_plugin(plugin_ref)
settings = PluginSettingsService(project)
# create the new profile for this plugin
name = payload["name"]
profile = plugin.add_profile(slugify(name), label=name)
config.update_plugin(plugin)
profile_config = settings.profile_with_config(db.session,
plugin,
profile,
redacted=True)
freeze_profile_config_keys(profile_config)
return jsonify(profile_config)
def update_files(self):
"""
Update the files managed by Meltano inside the current project.
"""
click.secho("Updating files managed by plugins...", fg="blue")
file_plugins = ConfigService(self.project).get_files()
if not file_plugins:
click.echo("Nothing to update")
return
install_plugins(self.project,
file_plugins,
reason=PluginInstallReason.UPGRADE)
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
def __init__(
self,
elt_context: ELTContext,
config_service: ConfigService = None,
connection_service: ConnectionService = None,
**config,
):
self.context = elt_context
self.config = config
self.config_service = config_service or ConfigService(
elt_context.project)
self.connection_service = connection_service or ConnectionService(
elt_context)
self.tap_config_dir = Path(
config.get("tap_config_dir", "/etc/singer/tap"))
self.target_config_dir = Path(
config.get("target_config_dir", "/etc/singer/target"))
def __init__(
self,
project,
config_service: ConfigService = None,
plugin_discovery_service: PluginDiscoveryService = None,
show_hidden=True,
env_override={},
config_override={},
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.discovery_service = plugin_discovery_service or PluginDiscoveryService(
project
)
self.show_hidden = show_hidden
self.env_override = env_override
self.config_override = config_override
self._env = None
def __init__(
self,
project: Project,
config_service: ConfigService = None,
plugin_settings_service: PluginSettingsService = None,
plugin_discovery_service: PluginDiscoveryService = None,
):
self.project = project
self.config_service = config_service or ConfigService(project)
self.plugin_discovery_service = (
plugin_discovery_service
or PluginDiscoveryService(project, config_service=config_service))
self.plugin_settings_service = plugin_settings_service or PluginSettingsService(
project,
config_service=config_service,
plugin_discovery_service=plugin_discovery_service,
)
self._extractor = None
self._loader = None
self._job = None
def config_service(project):
return ConfigService(project, use_cache=False)
class DbtWorker(threading.Thread):
def __init__(self, project: Project, loader: str, loop=None):
super().__init__()
self.project = project
self.loader = loader
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self.observer = None
self._plugin = None
self._loop = loop or asyncio.get_event_loop()
@property
def transform_dir(self):
return self.project.root_dir("transform")
def setup_observer(self, queue):
# write every FS events in the Queue
event_handler = DbtEventHandler(queue)
observer = Observer()
observer.schedule(event_handler,
str(self.transform_dir),
recursive=True)
return observer
async def process(self, session):
dbt_service = DbtService(self.project)
while True:
# drain the queue
while not self._queue.empty():
self._queue.get_nowait()
self._queue.task_done()
# trigger the task
try:
loader = self.config_service.find_plugin(self.loader)
await dbt_service.docs(session, loader, "generate")
except PluginMissingError as err:
logging.warning(
f"Could not generate dbt docs: '{str(err)}' is missing.")
except:
pass
# wait for the next trigger
logging.info("Awaiting task")
await self._queue.get()
self._queue.task_done()
# wait for debounce
await asyncio.sleep(5)
def start(self):
try:
self._queue = asyncio.Queue(maxsize=1, loop=self._loop)
self.observer = self.setup_observer(self._queue)
self.observer.start()
super().start()
except OSError as err:
# most probably INotify being full
logging.warning(f"DbtWorker failed: INotify limit reached: {err}")
def run(self):
try:
self._plugin = self.config_service.find_plugin("dbt")
except PluginMissingError as err:
self._plugin = self.add_service.add(PluginType.TRANSFORMERS, "dbt")
self.install_service.install_plugin(self._plugin)
_, Session = project_engine(self.project)
try:
session = Session()
# TODO: this blocks the loop, we should probaly return a `Task` instance from
# this function and let the caller schedule it on the loop
# This class would not have to be a Thread an thus it could simplify the
# handling of such cases in the future
logging.info(
f"Auto-generating dbt docs for in '{self.transform_dir}' for {self.loader}"
)
self._loop.run_until_complete(self.process(session))
finally:
session.close()
def stop(self):
if self.observer:
self.observer.stop()
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
plugins = []
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
plugin = add_plugin(project,
PluginType.EXTRACTORS,
extractor,
add_service=add_service)
plugins.append(plugin)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.LOADERS,
loader,
add_service=add_service)
plugins.append(plugin)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.TRANSFORMERS,
"dbt",
add_service=add_service)
plugins.append(plugin)
discovery_service = PluginDiscoveryService(project)
extractor_plugin_def = discovery_service.find_plugin(
PluginType.EXTRACTORS, extractor)
try:
transform_plugin = config_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service = TransformAddService(project)
transform_add_service.update_dbt_project(transform_plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
transform_plugin_def = discovery_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
click.secho(
f"Transform '{transform_plugin_def.name}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(
project,
PluginType.TRANSFORMS,
transform_plugin_def.name,
add_service=add_service,
)
plugins.append(plugin)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
return install_plugins(project, plugins)
class AirflowWorker(threading.Thread):
def __init__(self, project: Project):
super().__init__(name="AirflowWorker")
self.project = project
self.config_service = ConfigService(project)
self._plugin = None
self.pid_file = PIDFile(
self.project.run_dir("airflow", "scheduler.pid"))
def kill_stale_workers(self):
process = None
try:
process = self.pid_file.process
except UnknownProcessError:
pass
if process is not None:
logging.debug(
f"Process {process} is running, possibly stale, terminating it."
)
process.terminate()
def on_terminate(process):
logging.info(
f"Process {process} ended with exit code {process.returncode}"
)
_gone, alive = psutil.wait_procs([process],
timeout=5,
callback=on_terminate)
# kill the rest
for process in alive:
process.kill()
try:
self.pid_file.unlink()
except:
pass
def start_all(self):
_, Session = project_engine(self.project)
logs_path = self.project.run_dir("airflow", "logs", "scheduler.log")
try:
session = Session()
invoker = invoker_factory(self.project,
self._plugin,
prepare_with_session=session)
with logs_path.open("w") as logs_file:
scheduler = invoker.invoke("scheduler",
"--pid",
str(self.pid_file),
stdout=logs_file)
self.pid_file.write_pid(scheduler.pid)
finally:
session.close()
def run(self):
self._plugin = self.config_service.find_plugin("airflow")
self.kill_stale_workers()
self.start_all()
def stop(self):
self.kill_stale_workers()
def config_service(project):
return ConfigService(project)
def subject(self, project):
make_meltano_yml(project)
make_database_yml(project)
return ConfigService(project)
