def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type)
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_service.add_related(plugin)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
related_plugins.reverse()
install_service = PluginInstallService(project,
plugins_service=plugins_service)
install_status = install_service.install_plugins(
related_plugins, reason=PluginInstallReason.ADD)
for error in install_status["errors"]:
raise PluginInstallError(error["message"])
return jsonify([plugin.canonical() for plugin in related_plugins])
def add():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
add_service = ProjectAddService(project)
plugin = add_service.add(plugin_type, plugin_name)
return jsonify(plugin.canonical())
def __init__(self, project: Project):
super().__init__(name="AirflowWorker")
self.project = project
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self._plugin = None
self._webserver = None
self._scheduler = None
def __init__(self, project: Project, loader: str, loop=None):
super().__init__()
self.project = project
self.loader = loader
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self.observer = None
self._plugin = None
self._loop = loop or asyncio.get_event_loop()
def add_related_plugins(project,
plugins,
add_service: ProjectAddService,
plugin_types=list(PluginType)):
discovery_service = PluginDiscoveryService(project)
added_plugins = []
for plugin_install in plugins:
try:
plugin_def = discovery_service.find_plugin(plugin_install.type,
plugin_install.name)
except PluginNotFoundError:
continue
related_plugins = add_service.add_related(plugin_def,
plugin_types=plugin_types)
for related_plugin in related_plugins:
if related_plugin.should_add_to_file(project):
click.secho(
f"Added related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project",
fg="green",
)
else:
click.secho(
f"Adding related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project...",
fg="green",
)
added_plugins.extend(related_plugins)
return added_plugins
def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
config_service = ConfigService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = config_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
plugins = list(config_service.plugins())
if include_related:
add_service = ProjectAddService(project, config_service=config_service)
related_plugins = add_related_plugins(project, plugins, add_service=add_service)
plugins.extend(related_plugins)
click.echo(f"Installing {len(plugins)} plugins...")
success = install_plugins(project, plugins)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_install()
if not success:
raise click.Abort()
def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
plugins_service = ProjectPluginsService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = plugins_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
plugins = list(plugins_service.plugins())
if include_related:
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_related_plugins(project, plugins, add_service=add_service)
plugins.extend(related_plugins)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
plugins.reverse()
click.echo(f"Installing {len(plugins)} plugins...")
success = install_plugins(project, plugins)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_install()
if not success:
raise CliError("Failed to install plugin(s)")
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(add_service, project, PluginType.EXTRACTORS, extractor)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
add_plugin(add_service, project, PluginType.LOADERS, loader)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
add_plugin(add_service, project, PluginType.TRANSFORMERS, "dbt")
transform_add_service = TransformAddService(project)
try:
# the extractor name should match the transform name
plugin = config_service.find_plugin(
extractor, plugin_type=PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service.update_dbt_project(plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
PluginDiscoveryService(project).find_plugin(
PluginType.TRANSFORMS, extractor)
click.secho(
f"Transform '{extractor}' is missing, trying to install it...",
fg="yellow",
)
add_transform(project, extractor)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
# We use the DiscoveryService rather than the ConfigService because the
# plugin may not actually be installed yet at this point.
discovery = PluginDiscoveryService(project)
plugin = discovery.find_plugin(plugin_type, plugin_name)
add_service = ProjectAddService(project)
related_plugins = add_service.add_related(plugin)
install_service = PluginInstallService(project)
install_service.install_plugins(related_plugins)
return jsonify([plugin.canonical() for plugin in related_plugins])
def add(ctx, project, custom):
if custom:
if ctx.invoked_subcommand in (
"transformer",
"transform",
"orchestrator",
"connections",
):
click.secho(f"--custom is not supported for {ctx.invoked_subcommand}")
raise click.Abort()
ctx.obj["add_service"] = ProjectAddCustomService(project)
else:
ctx.obj["add_service"] = ProjectAddService(project)
def add_transform(project: Project, plugin_name: str):
try:
project_add_service = ProjectAddService(project)
plugin = project_add_service.add(PluginType.TRANSFORMS, plugin_name)
click.secho(
f"Added transform '{plugin_name}' to your Meltano project.", fg="green"
)
# Add repo to my-test-project/transform/packages.yml
transform_add_service = TransformAddService(project)
transform_add_service.add_to_packages(plugin)
click.secho(
f"Added transform '{plugin_name}' to your dbt packages.", fg="green"
)
# Add model and vars to my-test-project/transform/dbt_project.yml
transform_add_service.update_dbt_project(plugin)
click.secho(
f"Added transform '{plugin_name}' to your dbt_project.yml.", fg="green"
)
click.secho(f"Installed '{plugin_name}'.", fg="green")
except (PluginNotSupportedException, PluginNotFoundError):
click.secho(f"Error: transform '{plugin_name}' is not supported.", fg="red")
raise click.Abort()
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
def add(ctx, project, plugin_type, plugin_name, **flags):
plugin_type = PluginType.from_cli_argument(plugin_type)
plugin_names = plugin_name # nargs=-1
if flags["custom"]:
if plugin_type in (
PluginType.TRANSFORMERS,
PluginType.TRANSFORMS,
PluginType.ORCHESTRATORS,
):
click.secho(
f"--custom is not supported for {ctx.invoked_subcommand}")
raise click.Abort()
add_service = ProjectAddCustomService(project)
else:
add_service = ProjectAddService(project)
plugins = [
add_plugin(project, plugin_type, plugin_name, add_service=add_service)
for plugin_name in plugin_names
]
related_plugin_types = [PluginType.FILES]
if flags["include_related"]:
related_plugin_types = list(PluginType)
related_plugins = add_related_plugins(project,
plugins,
add_service=add_service,
plugin_types=related_plugin_types)
plugins.extend(related_plugins)
success = install_plugins(project, plugins, reason=PluginInstallReason.ADD)
for plugin in plugins: # TODO: Only works on Plugin from discovery...
docs_link = plugin._extras.get("docs")
if docs_link:
click.echo(
f"For more details about {plugin.type.descriptor} '{plugin.name}', visit {docs_link}"
)
if not success:
raise click.Abort()
def add(ctx, project, plugin_type, plugin_name, **flags):
if flags["custom"]:
if plugin_type in ("transformer", "transform", "orchestrator"):
click.secho(
f"--custom is not supported for {ctx.invoked_subcommand}")
raise click.Abort()
add_service = ProjectAddCustomService(project)
else:
add_service = ProjectAddService(project)
add_plugin(
add_service,
project,
PluginType(f"{plugin_type}s"),
plugin_name,
include_related=flags["include_related"],
)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name)
def add_plugin(
project: Project,
plugin_type: PluginType,
plugin_name: str,
add_service: ProjectAddService,
):
try:
plugin = add_service.add(plugin_type, plugin_name)
if plugin.should_add_to_file(project):
click.secho(
f"Added {plugin_type.descriptor} '{plugin_name}' to your Meltano project",
fg="green",
)
else:
click.secho(
f"Adding {plugin_type.descriptor} '{plugin_name}' to your Meltano project...",
fg="green",
)
except PluginAlreadyAddedException as err:
click.secho(
f"{plugin_type.descriptor.capitalize()} '{plugin_name}' is already in your Meltano project",
fg="yellow",
err=True,
)
plugin = err.plugin
except (PluginNotSupportedException, PluginNotFoundError) as err:
click.secho(
f"Error: {plugin_type.descriptor} '{plugin_name}' is not known to Meltano",
fg="red",
)
raise click.Abort()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name)
return plugin
class AirflowWorker(threading.Thread):
def __init__(self, project: Project):
super().__init__(name="AirflowWorker")
self.project = project
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self._plugin = None
self._webserver = None
self._scheduler = None
def kill_stale_workers(self):
stale_workers = []
workers_pid_files = map(self.pid_file, ("webserver", "scheduler"))
for pid_file in workers_pid_files:
try:
stale_workers.append(pid_file.process)
except UnknownProcessError:
pass
def on_terminate(process):
logging.info(
f"Process {process} ended with exit code {process.returncode}")
for process in stale_workers:
logging.debug(f"Process {process} is stale, terminating it.")
process.terminate()
gone, alive = psutil.wait_procs(stale_workers,
timeout=5,
callback=on_terminate)
# kill the rest
for process in alive:
process.kill()
for pid_file in workers_pid_files:
try:
pid_file.unlink()
except:
pass
def start_all(self):
_, Session = project_engine(self.project)
logs_dir = self.project.run_dir("airflow", "logs")
try:
session = Session()
invoker = invoker_factory(self.project,
self._plugin,
prepare_with_session=session)
# fmt: off
with logs_dir.joinpath("webserver.log").open("w") as webserver, \
logs_dir.joinpath("scheduler.log").open("w") as scheduler:
self._webserver = invoker.invoke("webserver",
"-w",
"1",
stdout=webserver)
self._scheduler = invoker.invoke("scheduler", stdout=scheduler)
self.pid_file("webserver").write_pid(self._webserver.pid)
self.pid_file("scheduler").write_pid(self._scheduler.pid)
# fmt: on
# Time padding for server initialization so UI iframe displays as expected
# (iteration potential on approach but following UIAvailableWorker sleep approach)
time.sleep(2)
finally:
session.close()
def pid_file(self, name) -> PIDFile:
return PIDFile(self.project.run_dir("airflow", f"{name}.pid"))
def run(self):
try:
self._plugin = self.config_service.find_plugin("airflow")
except PluginMissingError as err:
self._plugin = self.add_service.add(PluginType.ORCHESTRATORS,
"airflow")
self.install_service.install_plugin(self._plugin)
self.kill_stale_workers()
self.start_all()
def stop(self):
self.kill_stale_workers()
def project_add_service(project, project_plugins_service):
return ProjectAddService(project, plugins_service=project_plugins_service)
def add(
ctx,
project,
plugin_type,
plugin_name,
inherit_from=None,
variant=None,
as_name=None,
**flags,
):
"""Add a plugin to your project."""
plugin_type = PluginType.from_cli_argument(plugin_type)
plugin_names = plugin_name # nargs=-1
if as_name:
# `add <type> <inherit-from> --as <name>``
# is equivalent to:
# `add <type> <name> --inherit-from <inherit-from>``
inherit_from = plugin_names[0]
plugin_names = [as_name]
plugins_service = ProjectPluginsService(project)
if flags["custom"]:
if plugin_type in (
PluginType.TRANSFORMERS,
PluginType.TRANSFORMS,
PluginType.ORCHESTRATORS,
):
raise CliError(f"--custom is not supported for {plugin_type}")
add_service = ProjectAddService(project, plugins_service=plugins_service)
plugins = [
add_plugin(
project,
plugin_type,
plugin_name,
inherit_from=inherit_from,
variant=variant,
custom=flags["custom"],
add_service=add_service,
) for plugin_name in plugin_names
]
related_plugin_types = [PluginType.FILES]
if flags["include_related"]:
related_plugin_types = list(PluginType)
related_plugins = add_related_plugins(project,
plugins,
add_service=add_service,
plugin_types=related_plugin_types)
plugins.extend(related_plugins)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
plugins.reverse()
success = install_plugins(project, plugins, reason=PluginInstallReason.ADD)
if not success:
raise CliError("Failed to install plugin(s)")
_print_plugins(plugins)
class DbtWorker(threading.Thread):
def __init__(self, project: Project, loader: str, loop=None):
super().__init__()
self.project = project
self.loader = loader
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self.observer = None
self._plugin = None
self._loop = loop or asyncio.get_event_loop()
@property
def transform_dir(self):
return self.project.root_dir("transform")
def setup_observer(self, queue):
# write every FS events in the Queue
event_handler = DbtEventHandler(queue)
observer = Observer()
observer.schedule(event_handler,
str(self.transform_dir),
recursive=True)
return observer
async def process(self, session):
dbt_service = DbtService(self.project)
while True:
# drain the queue
while not self._queue.empty():
self._queue.get_nowait()
self._queue.task_done()
# trigger the task
try:
loader = self.config_service.find_plugin(self.loader)
await dbt_service.docs(session, loader, "generate")
except PluginMissingError as err:
logging.warning(
f"Could not generate dbt docs: '{str(err)}' is missing.")
except:
pass
# wait for the next trigger
logging.info("Awaiting task")
await self._queue.get()
self._queue.task_done()
# wait for debounce
await asyncio.sleep(5)
def start(self):
try:
self._queue = asyncio.Queue(maxsize=1, loop=self._loop)
self.observer = self.setup_observer(self._queue)
self.observer.start()
super().start()
except OSError as err:
# most probably INotify being full
logging.warning(f"DbtWorker failed: INotify limit reached: {err}")
def run(self):
try:
self._plugin = self.config_service.find_plugin("dbt")
except PluginMissingError as err:
self._plugin = self.add_service.add(PluginType.TRANSFORMERS, "dbt")
self.install_service.install_plugin(self._plugin)
_, Session = project_engine(self.project)
try:
session = Session()
# TODO: this blocks the loop, we should probaly return a `Task` instance from
# this function and let the caller schedule it on the loop
# This class would not have to be a Thread an thus it could simplify the
# handling of such cases in the future
logging.info(
f"Auto-generating dbt docs for in '{self.transform_dir}' for {self.loader}"
)
self._loop.run_until_complete(self.process(session))
finally:
session.close()
def stop(self):
if self.observer:
self.observer.stop()
def project_add_service(project, config_service, plugin_discovery_service):
return ProjectAddService(
project,
config_service=config_service,
plugin_discovery_service=plugin_discovery_service,
)
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
plugins = []
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
plugin = add_plugin(project,
PluginType.EXTRACTORS,
extractor,
add_service=add_service)
plugins.append(plugin)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.LOADERS,
loader,
add_service=add_service)
plugins.append(plugin)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.TRANSFORMERS,
"dbt",
add_service=add_service)
plugins.append(plugin)
discovery_service = PluginDiscoveryService(project)
extractor_plugin_def = discovery_service.find_plugin(
PluginType.EXTRACTORS, extractor)
try:
transform_plugin = config_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service = TransformAddService(project)
transform_add_service.update_dbt_project(transform_plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
transform_plugin_def = discovery_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
click.secho(
f"Transform '{transform_plugin_def.name}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(
project,
PluginType.TRANSFORMS,
transform_plugin_def.name,
add_service=add_service,
)
plugins.append(plugin)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
return install_plugins(project, plugins)