def parse(self): self.parser = MeltanoAnalysisFileParser(self.project) try: self.topics = self.parser.parse() self.package_topics = self.parser.parse_packages() self._parsed = True except MeltanoAnalysisFileParserError as e: logging.warning(f"Failed to compile topics: {e}") raise e
class ProjectCompiler: """ Meltano projects have multiple .m5o files that need to be compiled. This component is responsible for the compilation of the whole Meltano project. """ def __init__(self, project): self.project = project self._parsed = False self._lock = threading.Lock() @property def source_dir(self): return self.project.model_dir() def parse(self): self.parser = MeltanoAnalysisFileParser(self.project) try: self.topics = self.parser.parse() self.package_topics = self.parser.parse_packages() self._parsed = True except MeltanoAnalysisFileParserError as e: logging.warning(f"Failed to compile topics: {e}") raise e @fasteners.locked def compile(self): try: self.parse() all_topics = self.topics + self.package_topics if all_topics: self.parser.compile(all_topics) logging.debug(f"Successfully compiled topics") except MeltanoAnalysisFileParserError as e: logging.warning(f"Failed to compile topics: {e}") raise e
def index(): project = Project.find() onlyfiles = [f for f in project.model_dir().iterdir() if f.is_file()] path = project.model_dir() dashboardsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Dashboard) reportsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Report) reportsFiles = reportsParser.parse() dashboardFiles = dashboardsParser.parse() sortedM5oFiles = { "dashboards": {"label": "Dashboards", "items": dashboardFiles}, "documents": {"label": "Documents", "items": []}, "topics": {"label": "Topics", "items": []}, "reports": {"label": "Reports", "items": reportsFiles}, "tables": {"label": "Tables", "items": []}, } onlydocs = project.model_dir().parent.glob("*.md") for d in onlydocs: file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict( d.relative_to(project.root), str(d.name) ) sortedM5oFiles["documents"]["items"].append(file_dict) for f in onlyfiles: filename, ext = os.path.splitext(f) if ext != ".m5o": continue # filename splittext twice occurs due to current *.type.extension convention (two dots) filename = filename.lower() filename, ext = os.path.splitext(filename) file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict( f.relative_to(project.root), filename ) if ext == ".topic": sortedM5oFiles["topics"]["items"].append(file_dict) if ext == ".table": sortedM5oFiles["tables"]["items"].append(file_dict) m5o_parser = MeltanoAnalysisFileParser(project) for package in m5o_parser.packages(): package_files = MeltanoAnalysisFileParser.package_files(package) sortedM5oFiles["topics"]["items"] += package_files["topics"]["items"] sortedM5oFiles["tables"]["items"] += package_files["tables"]["items"] if not len(sortedM5oFiles["topics"]["items"]): return jsonify( { "result": False, "errors": [{"message": "Missing topic file(s)", "file_name": "*"}], "files": [], } ) return jsonify(sortedM5oFiles)
def save_report(self, data): report_name = data["name"] # guard if it already exists existing_report = self.get_report_by_name(report_name) if existing_report: raise ReportAlreadyExistsError(existing_report) project = Project.find() slug = slugify(report_name) file_path = project.analyze_dir("reports", f"{slug}.report.m5o") data = MeltanoAnalysisFileParser.fill_base_m5o_dict( file_path, slug, data) data["version"] = ReportsHelper.VERSION with file_path.open("w") as f: json.dump(data, f) return data
def save_dashboard(self, data): dashboard_name = data["name"] # guard if it already exists existing_dashboard = self.get_dashboard_by_name(dashboard_name) if existing_dashboard: raise DashboardAlreadyExistsError(existing_dashboard) project = Project.find() slug = slugify(dashboard_name) file_path = project.analyze_dir("dashboards", f"{slug}.dashboard.m5o") data = MeltanoAnalysisFileParser.fill_base_m5o_dict( file_path, slug, data) data["version"] = DashboardsHelper.VERSION data["description"] = data["description"] or "" data["report_ids"] = [] with file_path.open("w") as f: json.dump(data, f) return data
def subject(self, project): return MeltanoAnalysisFileParser(project)