def invoke(project, plugin_name, plugin_args): _, Session = project_engine(project) try: session = Session() config_service = ConfigService(project) plugin = config_service.find_plugin(plugin_name) service = invoker_factory(project, plugin, prepare_with_session=session) handle = service.invoke(*plugin_args) exit_code = handle.wait() tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_invoke(plugin_name=plugin_name, plugin_args=" ".join(plugin_args)) sys.exit(exit_code) except Exception as err: logging.exception(err) click.secho(f"An error occured: {err}.", fg="red") raise click.Abort() from err finally: session.close()
def config(ctx, project, plugin_type, plugin_name, format): plugin_type = PluginType.from_cli_argument( plugin_type) if plugin_type else None config = ConfigService(project) plugin = config.find_plugin(plugin_name, plugin_type=plugin_type, configurable=True) _, Session = project_engine(project) session = Session() try: settings = PluginSettingsService(project).build(plugin) ctx.obj["settings"] = settings ctx.obj["session"] = session if ctx.invoked_subcommand is None: if format == "json": config = settings.as_config(session=session) print(json.dumps(config)) elif format == "env": for env, value in settings.as_env(session=session).items(): print(f"{env}={value}") finally: session.close()
def install_missing_plugins(project: Project, extractor: str, loader: str, transform: str): add_service = ProjectAddService(project) config_service = ConfigService(project) if transform != "only": try: config_service.find_plugin(extractor, plugin_type=PluginType.EXTRACTORS) except PluginMissingError: click.secho( f"Extractor '{extractor}' is missing, trying to install it...", fg="yellow", ) add_plugin(add_service, project, PluginType.EXTRACTORS, extractor) try: config_service.find_plugin(loader, plugin_type=PluginType.LOADERS) except PluginMissingError: click.secho( f"Loader '{loader}' is missing, trying to install it...", fg="yellow") add_plugin(add_service, project, PluginType.LOADERS, loader) if transform != "skip": try: config_service.find_plugin("dbt", plugin_type=PluginType.TRANSFORMERS) except PluginMissingError as e: click.secho( f"Transformer 'dbt' is missing, trying to install it...", fg="yellow") add_plugin(add_service, project, PluginType.TRANSFORMERS, "dbt") transform_add_service = TransformAddService(project) try: # the extractor name should match the transform name plugin = config_service.find_plugin( extractor, plugin_type=PluginType.TRANSFORMS) # Update dbt_project.yml in case the vars values have changed in meltano.yml transform_add_service.update_dbt_project(plugin) except PluginMissingError: try: # Check if there is a default transform for this extractor PluginDiscoveryService(project).find_plugin( PluginType.TRANSFORMS, extractor) click.secho( f"Transform '{extractor}' is missing, trying to install it...", fg="yellow", ) add_transform(project, extractor) except PluginNotFoundError: # There is no default transform for this extractor.. # Don't panic, everything is cool - just run custom transforms pass
def start(ctx, reload, bind_port, bind): project = ctx.obj["project"] tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_ui() workers = [] if not truthy(os.getenv("MELTANO_DISABLE_AIRFLOW", False)): try: config_service = ConfigService(project) config_service.find_plugin("airflow") workers.append(AirflowWorker(project)) except PluginMissingError: pass try: compiler_worker = MeltanoCompilerWorker(project) compiler_worker.compiler.compile() workers.append(compiler_worker) except Exception as e: logger.error(f"Initial compilation failed: {e}") workers.append(UIAvailableWorker("http://localhost:{bind_port}")) workers.append( APIWorker( project, f"{bind}:{bind_port}", reload=reload or os.getenv("FLASK_ENV") == "development", )) cleanup = start_workers(workers) def handle_terminate(signal, frame): cleanup() signal.signal(signal.SIGTERM, handle_terminate) logger.info("All workers started.")
def install(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() config_service = ConfigService(project) plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type) install_service = PluginInstallService(project) install_service.install_plugin(plugin) return jsonify(plugin.canonical())
def config(ctx, project, plugin_name, format): config = ConfigService(project) plugin = config.find_plugin(plugin_name) _, Session = project_engine(project) session = Session() settings = PluginSettingsService(project) ctx.obj["settings"] = settings ctx.obj["plugin"] = plugin ctx.obj["session"] = session if ctx.invoked_subcommand is None: if format == "json": print(settings.as_config(session, plugin)) if format == "env": for env, value in settings.as_env(session, plugin).items(): print(f"{env}={value}")
def install(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() compiler = ProjectCompiler(project) install_service = PluginInstallService(project) config_service = ConfigService(project) plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type) run_venv = install_service.create_venv(plugin) run_install_plugin = install_service.install_plugin(plugin) if plugin_type is PluginType.MODELS: try: compiler.compile() except Exception as e: pass return jsonify(plugin.canonical())
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() discovery = PluginDiscoveryService(project) target_plugin = discovery.find_plugin(plugin_type, plugin_name) config_service = ConfigService(project) add_service = ProjectAddService(project) install_service = PluginInstallService(project) ignored_types = [target_plugin.type, PluginType.TRANSFORMS] has_model = False batched = [] for plugin in discovery.plugins(): if plugin.namespace == target_plugin.namespace: if plugin.type not in ignored_types: add_service.add(plugin.type, plugin.name) plugin_install = config_service.find_plugin( plugin.name, plugin_type=plugin.type) batched.append(plugin_install.canonical()) run_venv = install_service.create_venv(plugin_install) run_install_plugin = install_service.install_plugin( plugin_install) if plugin.type is PluginType.MODELS: has_model = True if has_model: compiler = ProjectCompiler(project) try: compiler.compile() except Exception as e: pass return jsonify(batched)
def install_missing_plugins(project: Project, extractor: str, loader: str, transform: str): add_service = ProjectAddService(project) config_service = ConfigService(project) plugins = [] if transform != "only": try: config_service.find_plugin(extractor, plugin_type=PluginType.EXTRACTORS) except PluginMissingError: click.secho( f"Extractor '{extractor}' is missing, trying to install it...", fg="yellow", ) plugin = add_plugin(project, PluginType.EXTRACTORS, extractor, add_service=add_service) plugins.append(plugin) try: config_service.find_plugin(loader, plugin_type=PluginType.LOADERS) except PluginMissingError: click.secho( f"Loader '{loader}' is missing, trying to install it...", fg="yellow") plugin = add_plugin(project, PluginType.LOADERS, loader, add_service=add_service) plugins.append(plugin) if transform != "skip": try: config_service.find_plugin("dbt", plugin_type=PluginType.TRANSFORMERS) except PluginMissingError as e: click.secho( f"Transformer 'dbt' is missing, trying to install it...", fg="yellow") plugin = add_plugin(project, PluginType.TRANSFORMERS, "dbt", add_service=add_service) plugins.append(plugin) discovery_service = PluginDiscoveryService(project) extractor_plugin_def = discovery_service.find_plugin( PluginType.EXTRACTORS, extractor) try: transform_plugin = config_service.find_plugin_by_namespace( extractor_plugin_def.namespace, PluginType.TRANSFORMS) # Update dbt_project.yml in case the vars values have changed in meltano.yml transform_add_service = TransformAddService(project) transform_add_service.update_dbt_project(transform_plugin) except PluginMissingError: try: # Check if there is a default transform for this extractor transform_plugin_def = discovery_service.find_plugin_by_namespace( extractor_plugin_def.namespace, PluginType.TRANSFORMS) click.secho( f"Transform '{transform_plugin_def.name}' is missing, trying to install it...", fg="yellow", ) add_plugin( project, PluginType.TRANSFORMS, transform_plugin_def.name, add_service=add_service, ) plugins.append(plugin) except PluginNotFoundError: # There is no default transform for this extractor.. # Don't panic, everything is cool - just run custom transforms pass return install_plugins(project, plugins)
class AirflowWorker(threading.Thread): def __init__(self, project: Project): super().__init__(name="AirflowWorker") self.project = project self.config_service = ConfigService(project) self._plugin = None self.pid_file = PIDFile( self.project.run_dir("airflow", "scheduler.pid")) def kill_stale_workers(self): process = None try: process = self.pid_file.process except UnknownProcessError: pass if process is not None: logging.debug( f"Process {process} is running, possibly stale, terminating it." ) process.terminate() def on_terminate(process): logging.info( f"Process {process} ended with exit code {process.returncode}" ) _gone, alive = psutil.wait_procs([process], timeout=5, callback=on_terminate) # kill the rest for process in alive: process.kill() try: self.pid_file.unlink() except: pass def start_all(self): _, Session = project_engine(self.project) logs_path = self.project.run_dir("airflow", "logs", "scheduler.log") try: session = Session() invoker = invoker_factory(self.project, self._plugin, prepare_with_session=session) with logs_path.open("w") as logs_file: scheduler = invoker.invoke("scheduler", "--pid", str(self.pid_file), stdout=logs_file) self.pid_file.write_pid(scheduler.pid) finally: session.close() def run(self): self._plugin = self.config_service.find_plugin("airflow") self.kill_stale_workers() self.start_all() def stop(self): self.kill_stale_workers()
class DbtWorker(threading.Thread): def __init__(self, project: Project, loader: str, loop=None): super().__init__() self.project = project self.loader = loader self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self.observer = None self._plugin = None self._loop = loop or asyncio.get_event_loop() @property def transform_dir(self): return self.project.root_dir("transform") def setup_observer(self, queue): # write every FS events in the Queue event_handler = DbtEventHandler(queue) observer = Observer() observer.schedule(event_handler, str(self.transform_dir), recursive=True) return observer async def process(self, session): dbt_service = DbtService(self.project) while True: # drain the queue while not self._queue.empty(): self._queue.get_nowait() self._queue.task_done() # trigger the task try: loader = self.config_service.find_plugin(self.loader) await dbt_service.docs(session, loader, "generate") except PluginMissingError as err: logging.warning( f"Could not generate dbt docs: '{str(err)}' is missing.") except: pass # wait for the next trigger logging.info("Awaiting task") await self._queue.get() self._queue.task_done() # wait for debounce await asyncio.sleep(5) def start(self): try: self._queue = asyncio.Queue(maxsize=1, loop=self._loop) self.observer = self.setup_observer(self._queue) self.observer.start() super().start() except OSError as err: # most probably INotify being full logging.warning(f"DbtWorker failed: INotify limit reached: {err}") def run(self): try: self._plugin = self.config_service.find_plugin("dbt") except PluginMissingError as err: self._plugin = self.add_service.add(PluginType.TRANSFORMERS, "dbt") self.install_service.install_plugin(self._plugin) _, Session = project_engine(self.project) try: session = Session() # TODO: this blocks the loop, we should probaly return a `Task` instance from # this function and let the caller schedule it on the loop # This class would not have to be a Thread an thus it could simplify the # handling of such cases in the future logging.info( f"Auto-generating dbt docs for in '{self.transform_dir}' for {self.loader}" ) self._loop.run_until_complete(self.process(session)) finally: session.close() def stop(self): if self.observer: self.observer.stop()
class AirflowWorker(threading.Thread): def __init__(self, project: Project): super().__init__(name="AirflowWorker") self.project = project self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self._plugin = None self._webserver = None self._scheduler = None def kill_stale_workers(self): stale_workers = [] workers_pid_files = map(self.pid_file, ("webserver", "scheduler")) for pid_file in workers_pid_files: try: stale_workers.append(pid_file.process) except UnknownProcessError: pass def on_terminate(process): logging.info( f"Process {process} ended with exit code {process.returncode}") for process in stale_workers: logging.debug(f"Process {process} is stale, terminating it.") process.terminate() gone, alive = psutil.wait_procs(stale_workers, timeout=5, callback=on_terminate) # kill the rest for process in alive: process.kill() for pid_file in workers_pid_files: try: pid_file.unlink() except: pass def start_all(self): _, Session = project_engine(self.project) logs_dir = self.project.run_dir("airflow", "logs") try: session = Session() invoker = invoker_factory(self.project, self._plugin, prepare_with_session=session) # fmt: off with logs_dir.joinpath("webserver.log").open("w") as webserver, \ logs_dir.joinpath("scheduler.log").open("w") as scheduler: self._webserver = invoker.invoke("webserver", "-w", "1", stdout=webserver) self._scheduler = invoker.invoke("scheduler", stdout=scheduler) self.pid_file("webserver").write_pid(self._webserver.pid) self.pid_file("scheduler").write_pid(self._scheduler.pid) # fmt: on # Time padding for server initialization so UI iframe displays as expected # (iteration potential on approach but following UIAvailableWorker sleep approach) time.sleep(2) finally: session.close() def pid_file(self, name) -> PIDFile: return PIDFile(self.project.run_dir("airflow", f"{name}.pid")) def run(self): try: self._plugin = self.config_service.find_plugin("airflow") except PluginMissingError as err: self._plugin = self.add_service.add(PluginType.ORCHESTRATORS, "airflow") self.install_service.install_plugin(self._plugin) self.kill_stale_workers() self.start_all() def stop(self): self.kill_stale_workers()