def emit_mce(self, mce: MetadataChangeEvent) -> None: url = f"{self._gms_server}/entities?action=ingest" raw_mce_obj = mce.proposedSnapshot.to_obj() mce_obj = pre_json_transform(raw_mce_obj) snapshot_fqn = ( f"com.linkedin.metadata.snapshot.{mce.proposedSnapshot.RECORD_SCHEMA.name}" ) system_metadata_obj = {} if mce.systemMetadata is not None: system_metadata_obj = { "lastObserved": mce.systemMetadata.lastObserved, "runId": mce.systemMetadata.runId, } snapshot = { "entity": { "value": { snapshot_fqn: mce_obj } }, "systemMetadata": system_metadata_obj, } payload = json.dumps(snapshot) self._emit_generic(url, payload)
def emit_mcp( self, mcp: Union[MetadataChangeProposal, MetadataChangeProposalWrapper]) -> None: url = f"{self._gms_server}/aspects?action=ingestProposal" mcp_obj = pre_json_transform(mcp.to_obj()) payload = json.dumps({"proposal": mcp_obj}) self._emit_generic(url, payload)
def emit_usage(self, usageStats: UsageAggregation) -> None: url = f"{self._gms_server}/usageStats?action=batchIngest" raw_usage_obj = usageStats.to_obj() usage_obj = pre_json_transform(raw_usage_obj) snapshot = { "buckets": [ usage_obj, ] } payload = json.dumps(snapshot) self._emit_generic(url, payload)
def ingest(self): proposed_snapshot = self._build_snapshot(random.randint(1, 100)) snapshot_fqn = ( f"com.linkedin.metadata.snapshot.{proposed_snapshot.RECORD_SCHEMA.name}" ) self.client.post( "/entities?action=ingest", json.dumps({ "entity": { "value": { snapshot_fqn: pre_json_transform(proposed_snapshot.to_obj()) } } }), )
def main(url: str, id: int): proposed_snapshot = _build_snapshot(id) snapshot_fqn = ( f"com.linkedin.metadata.snapshot.{proposed_snapshot.RECORD_SCHEMA.name}" ) requests.post( f"{url}/entities?action=ingest", data=json.dumps({ "entity": { "value": { snapshot_fqn: pre_json_transform(proposed_snapshot.to_obj()) } } }), )
def ingest(self): global num_ingested if num_ingested >= self.total: return lock.acquire() id = num_ingested num_ingested += 1 lock.release() proposed_snapshot = self._build_snapshot(id) snapshot_fqn = ( f"com.linkedin.metadata.snapshot.{proposed_snapshot.RECORD_SCHEMA.name}" ) self.client.post( "/entities?action=ingest", json.dumps({ "entity": { "value": { snapshot_fqn: pre_json_transform(proposed_snapshot.to_obj()) } } }), )
def datahub_guid(obj: dict) -> str: obj_str = json.dumps(pre_json_transform(obj), separators=(",", ":"), sort_keys=True).encode("utf-8") datahub_guid = md5(obj_str).hexdigest() return datahub_guid
def _make_generic_aspect(codegen_obj: DictWrapper) -> GenericAspectClass: serialized = json.dumps(pre_json_transform(codegen_obj.to_obj())) return GenericAspectClass( value=serialized.encode(), contentType="application/json", )