def get_all_concepts(db: PartitionedDatabase) -> List[JsonDict]: with db.transaction() as tx: models = db.get_models_tx(tx) property_counts = db.get_property_counts_tx(tx, [model.id for model in models]) return [to_concept_dict(m, property_counts[m.id]) for m in models]
def publish_schema( db: PartitionedDatabase, tx: Transaction, config: PublishConfig, s3, file_manifests: List[FileManifest], proxy_relationship_names: List[RelationshipName], ) -> Tuple[ExportGraphSchema, FileManifest]: """ Export the schema of the partitioned database into a `GraphSchema` instance. """ schema_models: List[ExportModel] = [] schema_relationships: List[ExportModelRelationship] = [] log.info("Exporting graph schema") models: List[Model] = db.get_models_tx(tx) model_index: Dict[UUID, Model] = {m.id: m for m in models} for m in models: log.info(f"Building schema for model '{m.name}'") properties: List[ModelProperty] = db.get_properties_tx(tx, m) linked_properties: List[ModelRelationship] = list( db.get_outgoing_model_relationships_tx(tx, from_model=m, one_to_many=False)) publish_properties: List[ExportProperty] = [ ExportProperty.model_property( name=p.name, display_name=p.display_name, description=p.description, data_type=p.data_type, ) for p in properties ] + [ ExportProperty.linked_property( name=r.name, display_name=r.display_name, description=r.description, data_type=LinkedModelDataType( to=model_index[r.to].name, file=str(OutputFile.csv_for_model(m.name)), ), ) for r in sorted(linked_properties, key=lambda l: l.index or sys.maxsize) ] model = ExportModel( model=m, name=m.name, display_name=m.display_name, description=m.description, properties=publish_properties, ) schema_models.append(model) # If any packages exist in this dataset, add a special-cased "File" model if len(file_manifests) > 0: log.info(f"Building schema for proxy package model") proxy_package_model = ExportModel.package_proxy() # TODO: gracefully handle this case to avoid overwriting "files.csv" assert not any( m.name == proxy_package_model.name for m in schema_models ), (f"Cannot export package proxy schema model with name '{proxy_package_model.name}' - " f"a model '{m.name}' already exists. See https://app.clickup.com/t/102ndc for issue" ) schema_models.append(proxy_package_model) relationships = db.get_outgoing_model_relationships_tx(tx, one_to_many=True) for r in relationships: log.info(f"Building schema for relationship '{r.name}'") relationship = ExportModelRelationship( relationship=r, name=r.name, from_=model_index[r.from_].name, to=model_index[r.to].name, ) schema_relationships.append(relationship) for p in proxy_relationship_names: log.info(f"Building schema for proxy relationship '{p}'") relationship = ExportModelRelationship(relationship=None, name=p, from_="", to="") schema_relationships.append(relationship) schema = ExportGraphSchema(models=schema_models, relationships=schema_relationships) # Write "schema.json" to S3 # ====================================================================== schema_output_file = OutputFile.json_for_schema().with_prefix( os.path.join(config.s3_publish_key, METADATA)) s3.put_object( Bucket=config.s3_bucket, Key=str(schema_output_file), Body=schema.to_json(camel_case=True, pretty_print=True, drop_nulls=True), RequestPayer="requester", ) schema_manifest = schema_output_file.with_prefix(METADATA).as_manifest( size_of(s3, config.s3_bucket, schema_output_file)) return schema, schema_manifest