def add(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() add_service = ProjectAddService(project) plugin = add_service.add(plugin_type, plugin_name) return jsonify(plugin.canonical())
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() discovery = PluginDiscoveryService(project) target_plugin = discovery.find_plugin(plugin_type, plugin_name) config_service = ConfigService(project) add_service = ProjectAddService(project) install_service = PluginInstallService(project) ignored_types = [target_plugin.type, PluginType.TRANSFORMS] has_model = False batched = [] for plugin in discovery.plugins(): if plugin.namespace == target_plugin.namespace: if plugin.type not in ignored_types: add_service.add(plugin.type, plugin.name) plugin_install = config_service.find_plugin( plugin.name, plugin_type=plugin.type) batched.append(plugin_install.canonical()) run_venv = install_service.create_venv(plugin_install) run_install_plugin = install_service.install_plugin( plugin_install) if plugin.type is PluginType.MODELS: has_model = True if has_model: compiler = ProjectCompiler(project) try: compiler.compile() except Exception as e: pass return jsonify(batched)
def add_plugin( project: Project, plugin_type: PluginType, plugin_name: str, add_service: ProjectAddService, ): try: plugin = add_service.add(plugin_type, plugin_name) if plugin.should_add_to_file(project): click.secho( f"Added {plugin_type.descriptor} '{plugin_name}' to your Meltano project", fg="green", ) else: click.secho( f"Adding {plugin_type.descriptor} '{plugin_name}' to your Meltano project...", fg="green", ) except PluginAlreadyAddedException as err: click.secho( f"{plugin_type.descriptor.capitalize()} '{plugin_name}' is already in your Meltano project", fg="yellow", err=True, ) plugin = err.plugin except (PluginNotSupportedException, PluginNotFoundError) as err: click.secho( f"Error: {plugin_type.descriptor} '{plugin_name}' is not known to Meltano", fg="red", ) raise click.Abort() tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name) return plugin
def add_transform(project: Project, plugin_name: str): try: project_add_service = ProjectAddService(project) plugin = project_add_service.add(PluginType.TRANSFORMS, plugin_name) click.secho( f"Added transform '{plugin_name}' to your Meltano project.", fg="green" ) # Add repo to my-test-project/transform/packages.yml transform_add_service = TransformAddService(project) transform_add_service.add_to_packages(plugin) click.secho( f"Added transform '{plugin_name}' to your dbt packages.", fg="green" ) # Add model and vars to my-test-project/transform/dbt_project.yml transform_add_service.update_dbt_project(plugin) click.secho( f"Added transform '{plugin_name}' to your dbt_project.yml.", fg="green" ) click.secho(f"Installed '{plugin_name}'.", fg="green") except (PluginNotSupportedException, PluginNotFoundError): click.secho(f"Error: transform '{plugin_name}' is not supported.", fg="red") raise click.Abort()
class DbtWorker(threading.Thread): def __init__(self, project: Project, loader: str, loop=None): super().__init__() self.project = project self.loader = loader self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self.observer = None self._plugin = None self._loop = loop or asyncio.get_event_loop() @property def transform_dir(self): return self.project.root_dir("transform") def setup_observer(self, queue): # write every FS events in the Queue event_handler = DbtEventHandler(queue) observer = Observer() observer.schedule(event_handler, str(self.transform_dir), recursive=True) return observer async def process(self, session): dbt_service = DbtService(self.project) while True: # drain the queue while not self._queue.empty(): self._queue.get_nowait() self._queue.task_done() # trigger the task try: loader = self.config_service.find_plugin(self.loader) await dbt_service.docs(session, loader, "generate") except PluginMissingError as err: logging.warning( f"Could not generate dbt docs: '{str(err)}' is missing.") except: pass # wait for the next trigger logging.info("Awaiting task") await self._queue.get() self._queue.task_done() # wait for debounce await asyncio.sleep(5) def start(self): try: self._queue = asyncio.Queue(maxsize=1, loop=self._loop) self.observer = self.setup_observer(self._queue) self.observer.start() super().start() except OSError as err: # most probably INotify being full logging.warning(f"DbtWorker failed: INotify limit reached: {err}") def run(self): try: self._plugin = self.config_service.find_plugin("dbt") except PluginMissingError as err: self._plugin = self.add_service.add(PluginType.TRANSFORMERS, "dbt") self.install_service.install_plugin(self._plugin) _, Session = project_engine(self.project) try: session = Session() # TODO: this blocks the loop, we should probaly return a `Task` instance from # this function and let the caller schedule it on the loop # This class would not have to be a Thread an thus it could simplify the # handling of such cases in the future logging.info( f"Auto-generating dbt docs for in '{self.transform_dir}' for {self.loader}" ) self._loop.run_until_complete(self.process(session)) finally: session.close() def stop(self): if self.observer: self.observer.stop()
class AirflowWorker(threading.Thread): def __init__(self, project: Project): super().__init__(name="AirflowWorker") self.project = project self.config_service = ConfigService(project) self.add_service = ProjectAddService( project, config_service=self.config_service) self.install_service = PluginInstallService( project, config_service=self.config_service) self._plugin = None self._webserver = None self._scheduler = None def kill_stale_workers(self): stale_workers = [] workers_pid_files = map(self.pid_file, ("webserver", "scheduler")) for pid_file in workers_pid_files: try: stale_workers.append(pid_file.process) except UnknownProcessError: pass def on_terminate(process): logging.info( f"Process {process} ended with exit code {process.returncode}") for process in stale_workers: logging.debug(f"Process {process} is stale, terminating it.") process.terminate() gone, alive = psutil.wait_procs(stale_workers, timeout=5, callback=on_terminate) # kill the rest for process in alive: process.kill() for pid_file in workers_pid_files: try: pid_file.unlink() except: pass def start_all(self): _, Session = project_engine(self.project) logs_dir = self.project.run_dir("airflow", "logs") try: session = Session() invoker = invoker_factory(self.project, self._plugin, prepare_with_session=session) # fmt: off with logs_dir.joinpath("webserver.log").open("w") as webserver, \ logs_dir.joinpath("scheduler.log").open("w") as scheduler: self._webserver = invoker.invoke("webserver", "-w", "1", stdout=webserver) self._scheduler = invoker.invoke("scheduler", stdout=scheduler) self.pid_file("webserver").write_pid(self._webserver.pid) self.pid_file("scheduler").write_pid(self._scheduler.pid) # fmt: on # Time padding for server initialization so UI iframe displays as expected # (iteration potential on approach but following UIAvailableWorker sleep approach) time.sleep(2) finally: session.close() def pid_file(self, name) -> PIDFile: return PIDFile(self.project.run_dir("airflow", f"{name}.pid")) def run(self): try: self._plugin = self.config_service.find_plugin("airflow") except PluginMissingError as err: self._plugin = self.add_service.add(PluginType.ORCHESTRATORS, "airflow") self.install_service.install_plugin(self._plugin) self.kill_stale_workers() self.start_all() def stop(self): self.kill_stale_workers()