def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() plugins_service = ProjectPluginsService(project) plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type) add_service = ProjectAddService(project, plugins_service=plugins_service) related_plugins = add_service.add_related(plugin) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. related_plugins.reverse() install_service = PluginInstallService(project, plugins_service=plugins_service) install_status = install_service.install_plugins( related_plugins, reason=PluginInstallReason.ADD) for error in install_status["errors"]: raise PluginInstallError(error["message"]) return jsonify([plugin.canonical() for plugin in related_plugins])
def add(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() add_service = ProjectAddService(project) plugin = add_service.add(plugin_type, plugin_name) return jsonify(plugin.canonical())
def __init__(self, project: Project): super().__init__(name="AirflowWorker") self.project = project self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self._plugin = None self._webserver = None self._scheduler = None
def __init__(self, project: Project, loader: str, loop=None): super().__init__() self.project = project self.loader = loader self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self.observer = None self._plugin = None self._loop = loop or asyncio.get_event_loop()
def add_related_plugins(project, plugins, add_service: ProjectAddService, plugin_types=list(PluginType)): discovery_service = PluginDiscoveryService(project) added_plugins = [] for plugin_install in plugins: try: plugin_def = discovery_service.find_plugin(plugin_install.type, plugin_install.name) except PluginNotFoundError: continue related_plugins = add_service.add_related(plugin_def, plugin_types=plugin_types) for related_plugin in related_plugins: if related_plugin.should_add_to_file(project): click.secho( f"Added related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project", fg="green", ) else: click.secho( f"Adding related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project...", fg="green", ) added_plugins.extend(related_plugins) return added_plugins
def install(project, plugin_type, plugin_name, include_related): """ Installs all the dependencies of your project based on the meltano.yml file. Read more at https://www.meltano.com/docs/command-line-interface.html. """ config_service = ConfigService(project) if plugin_type: plugin_type = PluginType.from_cli_argument(plugin_type) plugins = config_service.get_plugins_of_type(plugin_type) if plugin_name: plugins = [p for p in plugins if p.name in plugin_name] else: plugins = list(config_service.plugins()) if include_related: add_service = ProjectAddService(project, config_service=config_service) related_plugins = add_related_plugins(project, plugins, add_service=add_service) plugins.extend(related_plugins) click.echo(f"Installing {len(plugins)} plugins...") success = install_plugins(project, plugins) tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_install() if not success: raise click.Abort()
def install(project, plugin_type, plugin_name, include_related): """ Installs all the dependencies of your project based on the meltano.yml file. Read more at https://www.meltano.com/docs/command-line-interface.html. """ plugins_service = ProjectPluginsService(project) if plugin_type: plugin_type = PluginType.from_cli_argument(plugin_type) plugins = plugins_service.get_plugins_of_type(plugin_type) if plugin_name: plugins = [p for p in plugins if p.name in plugin_name] else: plugins = list(plugins_service.plugins()) if include_related: add_service = ProjectAddService(project, plugins_service=plugins_service) related_plugins = add_related_plugins(project, plugins, add_service=add_service) plugins.extend(related_plugins) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. plugins.reverse() click.echo(f"Installing {len(plugins)} plugins...") success = install_plugins(project, plugins) tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_install() if not success: raise CliError("Failed to install plugin(s)")
def install_missing_plugins(project: Project, extractor: str, loader: str, transform: str): add_service = ProjectAddService(project) config_service = ConfigService(project) if transform != "only": try: config_service.find_plugin(extractor, plugin_type=PluginType.EXTRACTORS) except PluginMissingError: click.secho( f"Extractor '{extractor}' is missing, trying to install it...", fg="yellow", ) add_plugin(add_service, project, PluginType.EXTRACTORS, extractor) try: config_service.find_plugin(loader, plugin_type=PluginType.LOADERS) except PluginMissingError: click.secho( f"Loader '{loader}' is missing, trying to install it...", fg="yellow") add_plugin(add_service, project, PluginType.LOADERS, loader) if transform != "skip": try: config_service.find_plugin("dbt", plugin_type=PluginType.TRANSFORMERS) except PluginMissingError as e: click.secho( f"Transformer 'dbt' is missing, trying to install it...", fg="yellow") add_plugin(add_service, project, PluginType.TRANSFORMERS, "dbt") transform_add_service = TransformAddService(project) try: # the extractor name should match the transform name plugin = config_service.find_plugin( extractor, plugin_type=PluginType.TRANSFORMS) # Update dbt_project.yml in case the vars values have changed in meltano.yml transform_add_service.update_dbt_project(plugin) except PluginMissingError: try: # Check if there is a default transform for this extractor PluginDiscoveryService(project).find_plugin( PluginType.TRANSFORMS, extractor) click.secho( f"Transform '{extractor}' is missing, trying to install it...", fg="yellow", ) add_transform(project, extractor) except PluginNotFoundError: # There is no default transform for this extractor.. # Don't panic, everything is cool - just run custom transforms pass
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() # We use the DiscoveryService rather than the ConfigService because the # plugin may not actually be installed yet at this point. discovery = PluginDiscoveryService(project) plugin = discovery.find_plugin(plugin_type, plugin_name) add_service = ProjectAddService(project) related_plugins = add_service.add_related(plugin) install_service = PluginInstallService(project) install_service.install_plugins(related_plugins) return jsonify([plugin.canonical() for plugin in related_plugins])
def add(ctx, project, custom): if custom: if ctx.invoked_subcommand in ( "transformer", "transform", "orchestrator", "connections", ): click.secho(f"--custom is not supported for {ctx.invoked_subcommand}") raise click.Abort() ctx.obj["add_service"] = ProjectAddCustomService(project) else: ctx.obj["add_service"] = ProjectAddService(project)
def add_transform(project: Project, plugin_name: str): try: project_add_service = ProjectAddService(project) plugin = project_add_service.add(PluginType.TRANSFORMS, plugin_name) click.secho( f"Added transform '{plugin_name}' to your Meltano project.", fg="green" ) # Add repo to my-test-project/transform/packages.yml transform_add_service = TransformAddService(project) transform_add_service.add_to_packages(plugin) click.secho( f"Added transform '{plugin_name}' to your dbt packages.", fg="green" ) # Add model and vars to my-test-project/transform/dbt_project.yml transform_add_service.update_dbt_project(plugin) click.secho( f"Added transform '{plugin_name}' to your dbt_project.yml.", fg="green" ) click.secho(f"Installed '{plugin_name}'.", fg="green") except (PluginNotSupportedException, PluginNotFoundError): click.secho(f"Error: transform '{plugin_name}' is not supported.", fg="red") raise click.Abort()
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() discovery = PluginDiscoveryService(project) target_plugin = discovery.find_plugin(plugin_type, plugin_name) config_service = ConfigService(project) add_service = ProjectAddService(project) install_service = PluginInstallService(project) ignored_types = [target_plugin.type, PluginType.TRANSFORMS] has_model = False batched = [] for plugin in discovery.plugins(): if plugin.namespace == target_plugin.namespace: if plugin.type not in ignored_types: add_service.add(plugin.type, plugin.name) plugin_install = config_service.find_plugin( plugin.name, plugin_type=plugin.type) batched.append(plugin_install.canonical()) run_venv = install_service.create_venv(plugin_install) run_install_plugin = install_service.install_plugin( plugin_install) if plugin.type is PluginType.MODELS: has_model = True if has_model: compiler = ProjectCompiler(project) try: compiler.compile() except Exception as e: pass return jsonify(batched)
def add(ctx, project, plugin_type, plugin_name, **flags): plugin_type = PluginType.from_cli_argument(plugin_type) plugin_names = plugin_name # nargs=-1 if flags["custom"]: if plugin_type in ( PluginType.TRANSFORMERS, PluginType.TRANSFORMS, PluginType.ORCHESTRATORS, ): click.secho( f"--custom is not supported for {ctx.invoked_subcommand}") raise click.Abort() add_service = ProjectAddCustomService(project) else: add_service = ProjectAddService(project) plugins = [ add_plugin(project, plugin_type, plugin_name, add_service=add_service) for plugin_name in plugin_names ] related_plugin_types = [PluginType.FILES] if flags["include_related"]: related_plugin_types = list(PluginType) related_plugins = add_related_plugins(project, plugins, add_service=add_service, plugin_types=related_plugin_types) plugins.extend(related_plugins) success = install_plugins(project, plugins, reason=PluginInstallReason.ADD) for plugin in plugins: # TODO: Only works on Plugin from discovery... docs_link = plugin._extras.get("docs") if docs_link: click.echo( f"For more details about {plugin.type.descriptor} '{plugin.name}', visit {docs_link}" ) if not success: raise click.Abort()
def add(ctx, project, plugin_type, plugin_name, **flags): if flags["custom"]: if plugin_type in ("transformer", "transform", "orchestrator"): click.secho( f"--custom is not supported for {ctx.invoked_subcommand}") raise click.Abort() add_service = ProjectAddCustomService(project) else: add_service = ProjectAddService(project) add_plugin( add_service, project, PluginType(f"{plugin_type}s"), plugin_name, include_related=flags["include_related"], ) tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name)
def add_plugin( project: Project, plugin_type: PluginType, plugin_name: str, add_service: ProjectAddService, ): try: plugin = add_service.add(plugin_type, plugin_name) if plugin.should_add_to_file(project): click.secho( f"Added {plugin_type.descriptor} '{plugin_name}' to your Meltano project", fg="green", ) else: click.secho( f"Adding {plugin_type.descriptor} '{plugin_name}' to your Meltano project...", fg="green", ) except PluginAlreadyAddedException as err: click.secho( f"{plugin_type.descriptor.capitalize()} '{plugin_name}' is already in your Meltano project", fg="yellow", err=True, ) plugin = err.plugin except (PluginNotSupportedException, PluginNotFoundError) as err: click.secho( f"Error: {plugin_type.descriptor} '{plugin_name}' is not known to Meltano", fg="red", ) raise click.Abort() tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name) return plugin
class AirflowWorker(threading.Thread): def __init__(self, project: Project): super().__init__(name="AirflowWorker") self.project = project self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self._plugin = None self._webserver = None self._scheduler = None def kill_stale_workers(self): stale_workers = [] workers_pid_files = map(self.pid_file, ("webserver", "scheduler")) for pid_file in workers_pid_files: try: stale_workers.append(pid_file.process) except UnknownProcessError: pass def on_terminate(process): logging.info( f"Process {process} ended with exit code {process.returncode}") for process in stale_workers: logging.debug(f"Process {process} is stale, terminating it.") process.terminate() gone, alive = psutil.wait_procs(stale_workers, timeout=5, callback=on_terminate) # kill the rest for process in alive: process.kill() for pid_file in workers_pid_files: try: pid_file.unlink() except: pass def start_all(self): _, Session = project_engine(self.project) logs_dir = self.project.run_dir("airflow", "logs") try: session = Session() invoker = invoker_factory(self.project, self._plugin, prepare_with_session=session) # fmt: off with logs_dir.joinpath("webserver.log").open("w") as webserver, \ logs_dir.joinpath("scheduler.log").open("w") as scheduler: self._webserver = invoker.invoke("webserver", "-w", "1", stdout=webserver) self._scheduler = invoker.invoke("scheduler", stdout=scheduler) self.pid_file("webserver").write_pid(self._webserver.pid) self.pid_file("scheduler").write_pid(self._scheduler.pid) # fmt: on # Time padding for server initialization so UI iframe displays as expected # (iteration potential on approach but following UIAvailableWorker sleep approach) time.sleep(2) finally: session.close() def pid_file(self, name) -> PIDFile: return PIDFile(self.project.run_dir("airflow", f"{name}.pid")) def run(self): try: self._plugin = self.config_service.find_plugin("airflow") except PluginMissingError as err: self._plugin = self.add_service.add(PluginType.ORCHESTRATORS, "airflow") self.install_service.install_plugin(self._plugin) self.kill_stale_workers() self.start_all() def stop(self): self.kill_stale_workers()
def project_add_service(project, project_plugins_service): return ProjectAddService(project, plugins_service=project_plugins_service)
def add( ctx, project, plugin_type, plugin_name, inherit_from=None, variant=None, as_name=None, **flags, ): """Add a plugin to your project.""" plugin_type = PluginType.from_cli_argument(plugin_type) plugin_names = plugin_name # nargs=-1 if as_name: # `add <type> <inherit-from> --as <name>`` # is equivalent to: # `add <type> <name> --inherit-from <inherit-from>`` inherit_from = plugin_names[0] plugin_names = [as_name] plugins_service = ProjectPluginsService(project) if flags["custom"]: if plugin_type in ( PluginType.TRANSFORMERS, PluginType.TRANSFORMS, PluginType.ORCHESTRATORS, ): raise CliError(f"--custom is not supported for {plugin_type}") add_service = ProjectAddService(project, plugins_service=plugins_service) plugins = [ add_plugin( project, plugin_type, plugin_name, inherit_from=inherit_from, variant=variant, custom=flags["custom"], add_service=add_service, ) for plugin_name in plugin_names ] related_plugin_types = [PluginType.FILES] if flags["include_related"]: related_plugin_types = list(PluginType) related_plugins = add_related_plugins(project, plugins, add_service=add_service, plugin_types=related_plugin_types) plugins.extend(related_plugins) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. plugins.reverse() success = install_plugins(project, plugins, reason=PluginInstallReason.ADD) if not success: raise CliError("Failed to install plugin(s)") _print_plugins(plugins)
class DbtWorker(threading.Thread): def __init__(self, project: Project, loader: str, loop=None): super().__init__() self.project = project self.loader = loader self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self.observer = None self._plugin = None self._loop = loop or asyncio.get_event_loop() @property def transform_dir(self): return self.project.root_dir("transform") def setup_observer(self, queue): # write every FS events in the Queue event_handler = DbtEventHandler(queue) observer = Observer() observer.schedule(event_handler, str(self.transform_dir), recursive=True) return observer async def process(self, session): dbt_service = DbtService(self.project) while True: # drain the queue while not self._queue.empty(): self._queue.get_nowait() self._queue.task_done() # trigger the task try: loader = self.config_service.find_plugin(self.loader) await dbt_service.docs(session, loader, "generate") except PluginMissingError as err: logging.warning( f"Could not generate dbt docs: '{str(err)}' is missing.") except: pass # wait for the next trigger logging.info("Awaiting task") await self._queue.get() self._queue.task_done() # wait for debounce await asyncio.sleep(5) def start(self): try: self._queue = asyncio.Queue(maxsize=1, loop=self._loop) self.observer = self.setup_observer(self._queue) self.observer.start() super().start() except OSError as err: # most probably INotify being full logging.warning(f"DbtWorker failed: INotify limit reached: {err}") def run(self): try: self._plugin = self.config_service.find_plugin("dbt") except PluginMissingError as err: self._plugin = self.add_service.add(PluginType.TRANSFORMERS, "dbt") self.install_service.install_plugin(self._plugin) _, Session = project_engine(self.project) try: session = Session() # TODO: this blocks the loop, we should probaly return a `Task` instance from # this function and let the caller schedule it on the loop # This class would not have to be a Thread an thus it could simplify the # handling of such cases in the future logging.info( f"Auto-generating dbt docs for in '{self.transform_dir}' for {self.loader}" ) self._loop.run_until_complete(self.process(session)) finally: session.close() def stop(self): if self.observer: self.observer.stop()
def project_add_service(project, config_service, plugin_discovery_service): return ProjectAddService( project, config_service=config_service, plugin_discovery_service=plugin_discovery_service, )
def install_missing_plugins(project: Project, extractor: str, loader: str, transform: str): add_service = ProjectAddService(project) config_service = ConfigService(project) plugins = [] if transform != "only": try: config_service.find_plugin(extractor, plugin_type=PluginType.EXTRACTORS) except PluginMissingError: click.secho( f"Extractor '{extractor}' is missing, trying to install it...", fg="yellow", ) plugin = add_plugin(project, PluginType.EXTRACTORS, extractor, add_service=add_service) plugins.append(plugin) try: config_service.find_plugin(loader, plugin_type=PluginType.LOADERS) except PluginMissingError: click.secho( f"Loader '{loader}' is missing, trying to install it...", fg="yellow") plugin = add_plugin(project, PluginType.LOADERS, loader, add_service=add_service) plugins.append(plugin) if transform != "skip": try: config_service.find_plugin("dbt", plugin_type=PluginType.TRANSFORMERS) except PluginMissingError as e: click.secho( f"Transformer 'dbt' is missing, trying to install it...", fg="yellow") plugin = add_plugin(project, PluginType.TRANSFORMERS, "dbt", add_service=add_service) plugins.append(plugin) discovery_service = PluginDiscoveryService(project) extractor_plugin_def = discovery_service.find_plugin( PluginType.EXTRACTORS, extractor) try: transform_plugin = config_service.find_plugin_by_namespace( extractor_plugin_def.namespace, PluginType.TRANSFORMS) # Update dbt_project.yml in case the vars values have changed in meltano.yml transform_add_service = TransformAddService(project) transform_add_service.update_dbt_project(transform_plugin) except PluginMissingError: try: # Check if there is a default transform for this extractor transform_plugin_def = discovery_service.find_plugin_by_namespace( extractor_plugin_def.namespace, PluginType.TRANSFORMS) click.secho( f"Transform '{transform_plugin_def.name}' is missing, trying to install it...", fg="yellow", ) add_plugin( project, PluginType.TRANSFORMS, transform_plugin_def.name, add_service=add_service, ) plugins.append(plugin) except PluginNotFoundError: # There is no default transform for this extractor.. # Don't panic, everything is cool - just run custom transforms pass return install_plugins(project, plugins)