def parse(self):
self.parser = MeltanoAnalysisFileParser(self.project)
try:
self.topics = self.parser.parse()
self.package_topics = self.parser.parse_packages()
self._parsed = True
except MeltanoAnalysisFileParserError as e:
logging.warning(f"Failed to compile topics: {e}")
raise e
class ProjectCompiler:
"""
Meltano projects have multiple .m5o files that need to be
compiled.
This component is responsible for the compilation of
the whole Meltano project.
"""
def __init__(self, project):
self.project = project
self._parsed = False
self._lock = threading.Lock()
@property
def source_dir(self):
return self.project.model_dir()
def parse(self):
self.parser = MeltanoAnalysisFileParser(self.project)
try:
self.topics = self.parser.parse()
self.package_topics = self.parser.parse_packages()
self._parsed = True
except MeltanoAnalysisFileParserError as e:
logging.warning(f"Failed to compile topics: {e}")
raise e
@fasteners.locked
def compile(self):
try:
self.parse()
all_topics = self.topics + self.package_topics
if all_topics:
self.parser.compile(all_topics)
logging.debug(f"Successfully compiled topics")
except MeltanoAnalysisFileParserError as e:
logging.warning(f"Failed to compile topics: {e}")
raise e
def index():
project = Project.find()
onlyfiles = [f for f in project.model_dir().iterdir() if f.is_file()]
path = project.model_dir()
dashboardsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Dashboard)
reportsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Report)
reportsFiles = reportsParser.parse()
dashboardFiles = dashboardsParser.parse()
sortedM5oFiles = {
"dashboards": {"label": "Dashboards", "items": dashboardFiles},
"documents": {"label": "Documents", "items": []},
"topics": {"label": "Topics", "items": []},
"reports": {"label": "Reports", "items": reportsFiles},
"tables": {"label": "Tables", "items": []},
}
onlydocs = project.model_dir().parent.glob("*.md")
for d in onlydocs:
file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict(
d.relative_to(project.root), str(d.name)
)
sortedM5oFiles["documents"]["items"].append(file_dict)
for f in onlyfiles:
filename, ext = os.path.splitext(f)
if ext != ".m5o":
continue
# filename splittext twice occurs due to current *.type.extension convention (two dots)
filename = filename.lower()
filename, ext = os.path.splitext(filename)
file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict(
f.relative_to(project.root), filename
)
if ext == ".topic":
sortedM5oFiles["topics"]["items"].append(file_dict)
if ext == ".table":
sortedM5oFiles["tables"]["items"].append(file_dict)
m5o_parser = MeltanoAnalysisFileParser(project)
for package in m5o_parser.packages():
package_files = MeltanoAnalysisFileParser.package_files(package)
sortedM5oFiles["topics"]["items"] += package_files["topics"]["items"]
sortedM5oFiles["tables"]["items"] += package_files["tables"]["items"]
if not len(sortedM5oFiles["topics"]["items"]):
return jsonify(
{
"result": False,
"errors": [{"message": "Missing topic file(s)", "file_name": "*"}],
"files": [],
}
)
return jsonify(sortedM5oFiles)
def save_report(self, data):
report_name = data["name"]
# guard if it already exists
existing_report = self.get_report_by_name(report_name)
if existing_report:
raise ReportAlreadyExistsError(existing_report)
project = Project.find()
slug = slugify(report_name)
file_path = project.analyze_dir("reports", f"{slug}.report.m5o")
data = MeltanoAnalysisFileParser.fill_base_m5o_dict(
file_path, slug, data)
data["version"] = ReportsHelper.VERSION
with file_path.open("w") as f:
json.dump(data, f)
return data
def save_dashboard(self, data):
dashboard_name = data["name"]
# guard if it already exists
existing_dashboard = self.get_dashboard_by_name(dashboard_name)
if existing_dashboard:
raise DashboardAlreadyExistsError(existing_dashboard)
project = Project.find()
slug = slugify(dashboard_name)
file_path = project.analyze_dir("dashboards", f"{slug}.dashboard.m5o")
data = MeltanoAnalysisFileParser.fill_base_m5o_dict(
file_path, slug, data)
data["version"] = DashboardsHelper.VERSION
data["description"] = data["description"] or ""
data["report_ids"] = []
with file_path.open("w") as f:
json.dump(data, f)
return data
def subject(self, project):
return MeltanoAnalysisFileParser(project)