def validate_plugin_config(plugin: PluginRef, name, value, project: Project, settings: PluginSettingsService): setting_def = settings.find_setting(plugin, name) # we want to prevent the edition of protected settings from the UI if setting_def.protected: logging.warning("Cannot set a 'protected' configuration externally.") return False if setting_def.kind == "file" and value and value != "": uploads_directory = project.extract_dir(plugin.full_name) resolved_file_path = project.root_dir(value).resolve() if not str(resolved_file_path).startswith( str(uploads_directory) + "/"): logging.warning( "Cannot set a file configuration to a path outside the project directory" ) return False old_value, source = settings.get_value(db.session, plugin, name) if source in (PluginSettingValueSource.ENV, PluginSettingValueSource.MELTANO_YML): logging.warning( "Cannot override a configuration set in the environment or meltano.yml." ) return False return True
def project(self, test_dir, project_init_service): """This fixture returns the non-activated project.""" project = project_init_service.init(activate=False, add_discovery=True) yield project Project.deactivate() shutil.rmtree(project.root)
def test_activate(self, project): Project.deactivate() assert Project._default is None Project.activate(project) assert Project._default is project assert Project.find() is project
def project(self, project): Project.deactivate() monkeypatch = MonkeyPatch() monkeypatch.setenv(PROJECT_READONLY_ENV, "true") yield project monkeypatch.undo()
def test_activate(self, project): assert os.getenv("MELTANO_PROJECT") is None with open(".env", "w") as env: env.write(f"MELTANO_PROJECT={project.root}") # `Project.find()` always return the default instance Project.activate(project) assert os.getenv("MELTANO_PROJECT") == str(project.root) assert Project.find() is project
def job_state() -> Response: """ Endpoint for getting the status of N jobs """ project = Project.find() poll_payload = request.get_json() job_ids = poll_payload["job_ids"] jobs = [] for job_id in job_ids: finder = JobFinder(job_id) state_job = finder.latest(db.session) # Validate existence first as a job may not be queued yet as a result of # another prerequisite async process (dbt installation for example) if state_job: state_job_success = finder.latest_success(db.session) jobs.append({ "job_id": job_id, "is_complete": state_job.is_complete(), "has_error": state_job.has_error(), "started_at": state_job.started_at, "ended_at": state_job.ended_at, "has_ever_succeeded": state_job_success.is_success() if state_job_success else None, }) return jsonify({"jobs": jobs})
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() plugins_service = ProjectPluginsService(project) plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type) add_service = ProjectAddService(project, plugins_service=plugins_service) related_plugins = add_service.add_related(plugin) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. related_plugins.reverse() install_service = PluginInstallService(project, plugins_service=plugins_service) install_status = install_service.install_plugins( related_plugins, reason=PluginInstallReason.ADD) for error in install_status["errors"]: raise PluginInstallError(error["message"]) return jsonify([plugin.canonical() for plugin in related_plugins])
def cli(ctx, log_level, verbose): """ Get help at https://www.meltano.com/docs/command-line-interface.html """ if log_level: ProjectSettingsService.config_override["cli.log_level"] = log_level ctx.ensure_object(dict) ctx.obj["verbosity"] = verbose try: project = Project.find() setup_logging(project) readonly = ProjectSettingsService(project).get("project_readonly") if readonly: project.readonly = True if project.readonly: logger.debug("Project is read-only.") ctx.obj["project"] = project except ProjectNotFound as err: ctx.obj["project"] = None except IncompatibleVersionError as err: click.secho( "This Meltano project is incompatible with this version of `meltano`.", fg="yellow", ) click.echo( "For more details, visit http://meltano.com/docs/installation.html#upgrading-meltano-version" ) sys.exit(3)
def save_plugin_configuration(plugin_ref) -> Response: """ Endpoint for persisting a plugin configuration """ project = Project.find() payload = request.get_json() plugin = ConfigService(project).get_plugin(plugin_ref) # TODO iterate pipelines and save each, also set this connector's profile (reuse `pipelineInFocusIndex`?) settings = PluginSettingsService(project, show_hidden=False) for profile in payload: # select the correct profile name = profile["name"] plugin.use_profile(plugin.get_profile(name)) for name, value in profile["config"].items(): if not validate_plugin_config(plugin, name, value, project, settings): continue if value == "": settings.unset(db.session, plugin, name) else: settings.set(db.session, plugin, name, value) profiles = settings.profiles_with_config(db.session, plugin, redacted=True) for profile in profiles: freeze_profile_config_keys(profile) return jsonify(profiles)
def get_plugin_configuration(plugin_ref) -> Response: """ Endpoint for getting a plugin's configuration profiles """ project = Project.find() settings = PluginSettingsService(project, show_hidden=False) plugin = ConfigService(project).get_plugin(plugin_ref) discovery_service = PluginDiscoveryService(project) try: plugin_def = discovery_service.find_plugin(plugin.type, plugin.name) settings_group_validation = plugin_def.settings_group_validation except PluginNotFoundError: settings_group_validation = [] profiles = settings.profiles_with_config(db.session, plugin, redacted=True) for profile in profiles: freeze_profile_config_keys(profile) return jsonify({ "profiles": profiles, "settings": Canonical.as_canonical(settings.definitions(plugin)), "settings_group_validation": settings_group_validation, })
def decorated(*args, **kwargs): project = Project.find() settings_service = ProjectSettingsService(project) if settings_service.get("ui.readonly"): return ( jsonify({ "error": True, "code": "Meltano UI is running in read-only mode" }), HTTP_READONLY_CODE, ) if settings_service.get( "ui.anonymous_readonly") and current_user.is_anonymous: return ( jsonify({ "error": True, "code": "Meltano UI is running in read-only mode until you sign in", }), HTTP_READONLY_CODE, ) return f(*args, **kwargs)
def get_report_resource(self, resource_id, today=None): project = Project.find() reports_service = ReportsService(project) report = reports_service.get_report(resource_id) reports_helper = ReportsHelper() return reports_helper.get_report_with_query_results(report, today=today)
def get_sql(namespace, topic_name, design_name): sqlHelper = SqlHelper() m5oc = sqlHelper.get_m5oc_topic(namespace, topic_name) design = m5oc.design(design_name) incoming_json = request.get_json() sql_dict = sqlHelper.get_sql(design, incoming_json) outgoing_sql = sql_dict["sql"] aggregates = sql_dict["aggregates"] query_attributes = sql_dict["query_attributes"] base_dict = {"sql": outgoing_sql, "error": False} base_dict["query_attributes"] = query_attributes base_dict["aggregates"] = aggregates if not incoming_json["run"]: return jsonify(base_dict) # we need to find the pipeline that loaded the data for this model # this is running off the assumption that there is only one pipeline # that can load data for a specific model project = Project.find() schedule_service = ScheduleService(project) schedule = schedule_service.find_namespace_schedule( m5oc.content["plugin_namespace"]) results = sqlHelper.get_query_results(schedule.extractor, schedule.loader, schedule.transform, outgoing_sql) base_dict["results"] = results base_dict["empty"] = len(results) == 0 return jsonify(base_dict)
def job_log(job_id) -> Response: """ Endpoint for getting the most recent log generated by a job with job_id """ project = Project.find() try: log_service = JobLoggingService(project) log = log_service.get_latest_log(job_id) has_log_exceeded_max_size = False except SizeThresholdJobLogException as err: log = None has_log_exceeded_max_size = True finder = JobFinder(job_id) state_job = finder.latest(db.session) state_job_success = finder.latest_success(db.session) return jsonify({ "job_id": job_id, "log": log, "has_log_exceeded_max_size": has_log_exceeded_max_size, "has_error": state_job.has_error() if state_job else False, "started_at": state_job.started_at if state_job else None, "ended_at": state_job.ended_at if state_job else None, "trigger": state_job.trigger if state_job else None, "has_ever_succeeded": state_job_success.is_success() if state_job_success else None, })
def get_dashboards(self): project = Project.find() dashboardsParser = M5oCollectionParser( project.analyze_dir("dashboards"), M5oCollectionParserTypes.Dashboard) return dashboardsParser.parse()
def get_pipeline_schedules(): """ Endpoint for getting the pipeline schedules """ project = Project.find() schedule_service = ScheduleService(project) schedules = list(map(dict, schedule_service.schedules())) for schedule in schedules: finder = JobFinder(schedule["name"]) state_job = finder.latest(db.session) schedule["has_error"] = state_job.has_error() if state_job else False schedule["is_running"] = state_job.is_running() if state_job else False schedule["job_id"] = schedule["name"] schedule["started_at"] = state_job.started_at if state_job else None schedule["ended_at"] = state_job.ended_at if state_job else None schedule["trigger"] = state_job.trigger if state_job else None state_job_success = finder.latest_success(db.session) schedule["has_ever_succeeded"] = (state_job_success.is_success() if state_job_success else None) schedule["start_date"] = (schedule["start_date"].date().isoformat() if schedule["start_date"] else None) return jsonify(schedules)
def installed(): """Returns JSON of all installed plugins Fuses the discovery.yml data with meltano.yml data and sorts each type alphabetically by name """ project = Project.find() config = ConfigService(project) discovery = PluginDiscoveryService(project) installed_plugins = {} # merge definitions for plugin in sorted(config.plugins(), key=lambda x: x.name): try: definition = discovery.find_plugin(plugin.type, plugin.name) merged_plugin_definition = { **definition.canonical(), **plugin.canonical() } except PluginNotFoundError: merged_plugin_definition = {**plugin.canonical()} merged_plugin_definition.pop("settings", None) merged_plugin_definition.pop("select", None) if not plugin.type in installed_plugins: installed_plugins[plugin.type] = [] installed_plugins[plugin.type].append(merged_plugin_definition) return jsonify({ **project.meltano.canonical(), "plugins": installed_plugins })
def save_plugin_configuration(plugin_ref) -> Response: """ Endpoint for persisting a plugin configuration """ project = Project.find() payload = request.get_json() plugins_service = ProjectPluginsService(project) plugin = plugins_service.get_plugin(plugin_ref) settings = PluginSettingsService(project, plugin, plugins_service=plugins_service, show_hidden=False) config = payload.get("config", {}) for name, value in config.items(): if not validate_plugin_config(plugin, name, value, project, settings): continue if value == "": settings.unset(name, session=db.session) else: settings.set(name, value, session=db.session) return jsonify(get_config_with_metadata(settings))
def get_plugin_configuration(plugin_ref) -> Response: """ Endpoint for getting a plugin's configuration """ project = Project.find() plugins_service = ProjectPluginsService(project) plugin = plugins_service.get_plugin(plugin_ref) settings = PluginSettingsService(project, plugin, plugins_service=plugins_service, show_hidden=False) try: settings_group_validation = plugin.settings_group_validation except PluginNotFoundError: settings_group_validation = [] return jsonify({ **get_config_with_metadata(settings), "settings": Canonical.as_canonical(settings.definitions(extras=False)), "settings_group_validation": settings_group_validation, })
def run(): project = Project.find() schedule_payload = request.get_json() name = schedule_payload["name"] job_id = run_schedule(project, name) return jsonify({"job_id": job_id}), 202
def index(): project = Project.find() onlyfiles = [f for f in project.model_dir().iterdir() if f.is_file()] path = project.model_dir() dashboardsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Dashboard) reportsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Report) reportsFiles = reportsParser.parse() dashboardFiles = dashboardsParser.parse() sortedM5oFiles = { "dashboards": {"label": "Dashboards", "items": dashboardFiles}, "documents": {"label": "Documents", "items": []}, "topics": {"label": "Topics", "items": []}, "reports": {"label": "Reports", "items": reportsFiles}, "tables": {"label": "Tables", "items": []}, } onlydocs = project.model_dir().parent.glob("*.md") for d in onlydocs: file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict( d.relative_to(project.root), str(d.name) ) sortedM5oFiles["documents"]["items"].append(file_dict) for f in onlyfiles: filename, ext = os.path.splitext(f) if ext != ".m5o": continue # filename splittext twice occurs due to current *.type.extension convention (two dots) filename = filename.lower() filename, ext = os.path.splitext(filename) file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict( f.relative_to(project.root), filename ) if ext == ".topic": sortedM5oFiles["topics"]["items"].append(file_dict) if ext == ".table": sortedM5oFiles["tables"]["items"].append(file_dict) m5o_parser = MeltanoAnalysisFileParser(project) for package in m5o_parser.packages(): package_files = MeltanoAnalysisFileParser.package_files(package) sortedM5oFiles["topics"]["items"] += package_files["topics"]["items"] sortedM5oFiles["tables"]["items"] += package_files["tables"]["items"] if not len(sortedM5oFiles["topics"]["items"]): return jsonify( { "result": False, "errors": [{"message": "Missing topic file(s)", "file_name": "*"}], "files": [], } ) return jsonify(sortedM5oFiles)
def test_plugin_configuration(plugin_ref) -> Response: """ Endpoint for testing a plugin configuration's valid connection """ project = Project.find() payload = request.get_json() plugins_service = ProjectPluginsService(project) plugin = plugins_service.get_plugin(plugin_ref) settings = PluginSettingsService(project, plugin, plugins_service=plugins_service, show_hidden=False) config = payload.get("config", {}) valid_config = { name: value for name, value in config.items() if validate_plugin_config(plugin, name, value, project, settings) } settings.config_override = PluginSettingsService.unredact(valid_config) async def test_stream(tap_stream) -> bool: while not tap_stream.at_eof(): message = await tap_stream.readline() json_dict = json.loads(message) if json_dict["type"] == "RECORD": return True return False async def test_extractor(): process = None try: invoker = invoker_factory( project, plugin, plugins_service=plugins_service, plugin_settings_service=settings, ) with invoker.prepared(db.session): process = await invoker.invoke_async( stdout=asyncio.subprocess.PIPE) return await test_stream(process.stdout) except Exception as err: logging.debug(err) # if anything happens, this is not successful return False finally: try: if process: psutil.Process(process.pid).terminate() except Exception as err: logging.debug(err) loop = asyncio.get_event_loop() success = loop.run_until_complete(test_extractor()) return jsonify({"is_success": success}), 200
def download_job_log(job_id) -> Response: """ Endpoint for downloading a job log with job_id """ project = Project.find() log_service = JobLoggingService(project) return send_file(log_service.get_downloadable_log(job_id), mimetype="text/plain")
def __init__(self): project = Project.find() self.settings_file_path = project.root_dir("model", "database.settings.m5o") if not self.settings_file_path.is_file(): with open(self.settings_file_path, "w") as f: settings = {"settings": {"connections": []}} json.dump(settings, f)
def update_report(self, data): project = Project.find() file_path = project.analyze_dir("reports", f"{data['slug']}.report.m5o") with open(file_path, "w") as f: json.dump(data, f) return data
def test_init(self, cli_runner, tmp_path_factory, pushd): new_project_root = tmp_path_factory.mktemp("new_meltano_root") pushd(new_project_root) # there are no project actually assert Project._default is None with pytest.raises(ProjectNotFound): Project.find() # create one with the CLI cli_runner.invoke(cli, ["init", "test_project", "--no_usage_stats"]) pushd("test_project") project = Project.find() # Deactivate project Project.deactivate() files = ( project.root.joinpath(file).resolve() for file in ("meltano.yml", "README.md", ".gitignore", "requirements.txt") ) dirs = ( project.root.joinpath(dir) for dir in ( "model", "extract", "load", "transform", "analyze", "notebook", "orchestrate", ) ) for file in files: assert file.is_file() for dir in dirs: assert dir.is_dir() meltano_yml = project.root_dir("meltano.yml").read_text() assert "send_anonymous_usage_stats: false" in meltano_yml assert "project_id:" not in meltano_yml
def test_find(self, project, mkdtemp, monkeypatch): # defaults to the cwd found = Project.find(activate=False) assert found == project # or you can specify a path found = Project.find(project.root, activate=False) assert found == project # or set the MELTANO_PROJECT_ROOT env var with monkeypatch.context() as m: m.chdir(project.root.joinpath("model")) m.setenv(PROJECT_ROOT_ENV, "..") found = Project.find(activate=False) assert found == project # but it doens't recurse up, you have to be # at the meltano.yml level with pytest.raises(ProjectNotFound): Project.find(project.root.joinpath("model")) # and it fails if there isn't a meltano.yml with pytest.raises(ProjectNotFound): try: empty_dir = mkdtemp("meltano_empty_project") Project.find(empty_dir) finally: shutil.rmtree(empty_dir)
def get_topic(self, namespace, topic_name): project = Project.find() filename = enforce_secure_filename(f"{topic_name}.topic.m5oc") topic = project.run_dir("models", namespace, filename) with topic.open() as f: topic = json.load(f) return topic
def all(): project = Project.find() discovery = PluginDiscoveryService(project) ordered_plugins = {} for type, plugins in groupby(discovery.plugins(), key=lambda p: p.type): ordered_plugins[type] = [plugin.canonical() for plugin in plugins] return jsonify(ordered_plugins)
def __init__(self, project: Project): self.project = project self._output_dir = self.project.run_dir("models") self.topics = [] self.tables = [] self.packaged_topics = [] self.packaged_tables = [] self.required_topic_properties = ["name", "label", "designs"] self.required_design_properties = ["from", "label", "description"] self.required_join_properties = ["sql_on", "relationship"] self.required_table_properties = ["sql_table_name", "columns"] self.join_relationship_types = [ "one_to_one", "one_to_many", "many_to_one", "many_to_many", ] self.project = Project()