def emit_mce(self, mce: MetadataChangeEvent) -> None:
url = f"{self._gms_server}/entities?action=ingest"
raw_mce_obj = mce.proposedSnapshot.to_obj()
mce_obj = pre_json_transform(raw_mce_obj)
snapshot_fqn = (
f"com.linkedin.metadata.snapshot.{mce.proposedSnapshot.RECORD_SCHEMA.name}"
)
system_metadata_obj = {}
if mce.systemMetadata is not None:
system_metadata_obj = {
"lastObserved": mce.systemMetadata.lastObserved,
"runId": mce.systemMetadata.runId,
}
snapshot = {
"entity": {
"value": {
snapshot_fqn: mce_obj
}
},
"systemMetadata": system_metadata_obj,
}
payload = json.dumps(snapshot)
self._emit_generic(url, payload)
def emit_mcp(
self, mcp: Union[MetadataChangeProposal,
MetadataChangeProposalWrapper]) -> None:
url = f"{self._gms_server}/aspects?action=ingestProposal"
mcp_obj = pre_json_transform(mcp.to_obj())
payload = json.dumps({"proposal": mcp_obj})
self._emit_generic(url, payload)
def emit_usage(self, usageStats: UsageAggregation) -> None:
url = f"{self._gms_server}/usageStats?action=batchIngest"
raw_usage_obj = usageStats.to_obj()
usage_obj = pre_json_transform(raw_usage_obj)
snapshot = {
"buckets": [
usage_obj,
]
}
payload = json.dumps(snapshot)
self._emit_generic(url, payload)
def ingest(self):
proposed_snapshot = self._build_snapshot(random.randint(1, 100))
snapshot_fqn = (
f"com.linkedin.metadata.snapshot.{proposed_snapshot.RECORD_SCHEMA.name}"
)
self.client.post(
"/entities?action=ingest",
json.dumps({
"entity": {
"value": {
snapshot_fqn:
pre_json_transform(proposed_snapshot.to_obj())
}
}
}),
)
def main(url: str, id: int):
proposed_snapshot = _build_snapshot(id)
snapshot_fqn = (
f"com.linkedin.metadata.snapshot.{proposed_snapshot.RECORD_SCHEMA.name}"
)
requests.post(
f"{url}/entities?action=ingest",
data=json.dumps({
"entity": {
"value": {
snapshot_fqn:
pre_json_transform(proposed_snapshot.to_obj())
}
}
}),
)
def ingest(self):
global num_ingested
if num_ingested >= self.total:
return
lock.acquire()
id = num_ingested
num_ingested += 1
lock.release()
proposed_snapshot = self._build_snapshot(id)
snapshot_fqn = (
f"com.linkedin.metadata.snapshot.{proposed_snapshot.RECORD_SCHEMA.name}"
)
self.client.post(
"/entities?action=ingest",
json.dumps({
"entity": {
"value": {
snapshot_fqn:
pre_json_transform(proposed_snapshot.to_obj())
}
}
}),
)
def datahub_guid(obj: dict) -> str:
obj_str = json.dumps(pre_json_transform(obj),
separators=(",", ":"),
sort_keys=True).encode("utf-8")
datahub_guid = md5(obj_str).hexdigest()
return datahub_guid
def _make_generic_aspect(codegen_obj: DictWrapper) -> GenericAspectClass:
serialized = json.dumps(pre_json_transform(codegen_obj.to_obj()))
return GenericAspectClass(
value=serialized.encode(),
contentType="application/json",
)
