def add():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
add_service = ProjectAddService(project)
plugin = add_service.add(plugin_type, plugin_name)
return jsonify(plugin.canonical())
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
def add_plugin(
project: Project,
plugin_type: PluginType,
plugin_name: str,
add_service: ProjectAddService,
):
try:
plugin = add_service.add(plugin_type, plugin_name)
if plugin.should_add_to_file(project):
click.secho(
f"Added {plugin_type.descriptor} '{plugin_name}' to your Meltano project",
fg="green",
)
else:
click.secho(
f"Adding {plugin_type.descriptor} '{plugin_name}' to your Meltano project...",
fg="green",
)
except PluginAlreadyAddedException as err:
click.secho(
f"{plugin_type.descriptor.capitalize()} '{plugin_name}' is already in your Meltano project",
fg="yellow",
err=True,
)
plugin = err.plugin
except (PluginNotSupportedException, PluginNotFoundError) as err:
click.secho(
f"Error: {plugin_type.descriptor} '{plugin_name}' is not known to Meltano",
fg="red",
)
raise click.Abort()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name)
return plugin
def add_transform(project: Project, plugin_name: str):
try:
project_add_service = ProjectAddService(project)
plugin = project_add_service.add(PluginType.TRANSFORMS, plugin_name)
click.secho(
f"Added transform '{plugin_name}' to your Meltano project.", fg="green"
)
# Add repo to my-test-project/transform/packages.yml
transform_add_service = TransformAddService(project)
transform_add_service.add_to_packages(plugin)
click.secho(
f"Added transform '{plugin_name}' to your dbt packages.", fg="green"
)
# Add model and vars to my-test-project/transform/dbt_project.yml
transform_add_service.update_dbt_project(plugin)
click.secho(
f"Added transform '{plugin_name}' to your dbt_project.yml.", fg="green"
)
click.secho(f"Installed '{plugin_name}'.", fg="green")
except (PluginNotSupportedException, PluginNotFoundError):
click.secho(f"Error: transform '{plugin_name}' is not supported.", fg="red")
raise click.Abort()
class DbtWorker(threading.Thread):
def __init__(self, project: Project, loader: str, loop=None):
super().__init__()
self.project = project
self.loader = loader
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self.observer = None
self._plugin = None
self._loop = loop or asyncio.get_event_loop()
@property
def transform_dir(self):
return self.project.root_dir("transform")
def setup_observer(self, queue):
# write every FS events in the Queue
event_handler = DbtEventHandler(queue)
observer = Observer()
observer.schedule(event_handler,
str(self.transform_dir),
recursive=True)
return observer
async def process(self, session):
dbt_service = DbtService(self.project)
while True:
# drain the queue
while not self._queue.empty():
self._queue.get_nowait()
self._queue.task_done()
# trigger the task
try:
loader = self.config_service.find_plugin(self.loader)
await dbt_service.docs(session, loader, "generate")
except PluginMissingError as err:
logging.warning(
f"Could not generate dbt docs: '{str(err)}' is missing.")
except:
pass
# wait for the next trigger
logging.info("Awaiting task")
await self._queue.get()
self._queue.task_done()
# wait for debounce
await asyncio.sleep(5)
def start(self):
try:
self._queue = asyncio.Queue(maxsize=1, loop=self._loop)
self.observer = self.setup_observer(self._queue)
self.observer.start()
super().start()
except OSError as err:
# most probably INotify being full
logging.warning(f"DbtWorker failed: INotify limit reached: {err}")
def run(self):
try:
self._plugin = self.config_service.find_plugin("dbt")
except PluginMissingError as err:
self._plugin = self.add_service.add(PluginType.TRANSFORMERS, "dbt")
self.install_service.install_plugin(self._plugin)
_, Session = project_engine(self.project)
try:
session = Session()
# TODO: this blocks the loop, we should probaly return a `Task` instance from
# this function and let the caller schedule it on the loop
# This class would not have to be a Thread an thus it could simplify the
# handling of such cases in the future
logging.info(
f"Auto-generating dbt docs for in '{self.transform_dir}' for {self.loader}"
)
self._loop.run_until_complete(self.process(session))
finally:
session.close()
def stop(self):
if self.observer:
self.observer.stop()
class AirflowWorker(threading.Thread):
def __init__(self, project: Project):
super().__init__(name="AirflowWorker")
self.project = project
self.config_service = ConfigService(project)
self.add_service = ProjectAddService(
project, config_service=self.config_service)
self.install_service = PluginInstallService(
project, config_service=self.config_service)
self._plugin = None
self._webserver = None
self._scheduler = None
def kill_stale_workers(self):
stale_workers = []
workers_pid_files = map(self.pid_file, ("webserver", "scheduler"))
for pid_file in workers_pid_files:
try:
stale_workers.append(pid_file.process)
except UnknownProcessError:
pass
def on_terminate(process):
logging.info(
f"Process {process} ended with exit code {process.returncode}")
for process in stale_workers:
logging.debug(f"Process {process} is stale, terminating it.")
process.terminate()
gone, alive = psutil.wait_procs(stale_workers,
timeout=5,
callback=on_terminate)
# kill the rest
for process in alive:
process.kill()
for pid_file in workers_pid_files:
try:
pid_file.unlink()
except:
pass
def start_all(self):
_, Session = project_engine(self.project)
logs_dir = self.project.run_dir("airflow", "logs")
try:
session = Session()
invoker = invoker_factory(self.project,
self._plugin,
prepare_with_session=session)
# fmt: off
with logs_dir.joinpath("webserver.log").open("w") as webserver, \
logs_dir.joinpath("scheduler.log").open("w") as scheduler:
self._webserver = invoker.invoke("webserver",
"-w",
"1",
stdout=webserver)
self._scheduler = invoker.invoke("scheduler", stdout=scheduler)
self.pid_file("webserver").write_pid(self._webserver.pid)
self.pid_file("scheduler").write_pid(self._scheduler.pid)
# fmt: on
# Time padding for server initialization so UI iframe displays as expected
# (iteration potential on approach but following UIAvailableWorker sleep approach)
time.sleep(2)
finally:
session.close()
def pid_file(self, name) -> PIDFile:
return PIDFile(self.project.run_dir("airflow", f"{name}.pid"))
def run(self):
try:
self._plugin = self.config_service.find_plugin("airflow")
except PluginMissingError as err:
self._plugin = self.add_service.add(PluginType.ORCHESTRATORS,
"airflow")
self.install_service.install_plugin(self._plugin)
self.kill_stale_workers()
self.start_all()
def stop(self):
self.kill_stale_workers()
