def write_record_async(
self,
record_envelope: RecordEnvelope[MetadataChangeEvent],
write_callback: WriteCallback,
) -> None:
mce = record_envelope.record
obj = mce.to_obj()
if self.wrote_something:
self.file.write(",\n")
json.dump(obj, self.file, indent=4)
self.wrote_something = True
# record_string = str(record_envelope.record)
# metadata = record_envelope.metadata
# metadata["workunit-id"] = self.id
# out_line=f'{{"record": {record_string}, "metadata": {metadata}}}\n'
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
def _write_done_callback(
self,
record_envelope: RecordEnvelope,
write_callback: WriteCallback,
future: concurrent.futures.Future,
) -> None:
if future.cancelled():
self.report.report_failure({"error": "future was cancelled"})
write_callback.on_failure(record_envelope,
OperationalError("future was cancelled"),
{})
elif future.done():
e = future.exception()
if not e:
self.report.report_record_written(record_envelope)
start_time, end_time = future.result()
self.report.report_downstream_latency(start_time, end_time)
write_callback.on_success(record_envelope, {})
elif isinstance(e, OperationalError):
# only OperationalErrors should be ignored
if not self.treat_errors_as_warnings:
self.report.report_failure({
"error": e.message,
"info": e.info
})
else:
# trim exception stacktraces when reporting warnings
if "stackTrace" in e.info:
try:
e.info["stackTrace"] = "\n".join(
e.info["stackTrace"].split("\n")[0:2])
except Exception:
# ignore failures in trimming
pass
record = record_envelope.record
if isinstance(record, MetadataChangeProposalWrapper):
# include information about the entity that failed
entity_id = cast(MetadataChangeProposalWrapper,
record).entityUrn
e.info["id"] = entity_id
else:
entity_id = None
self.report.report_warning({
"warning": e.message,
"info": e.info
})
write_callback.on_failure(record_envelope, e, e.info)
else:
self.report.report_failure({"e": e})
write_callback.on_failure(record_envelope, Exception(e), {})
def write_record_async(
self,
record_envelope: RecordEnvelope[MetadataChangeEvent],
write_callback: WriteCallback,
):
headers = {"X-RestLi-Protocol-Version": "2.0.0"}
mce = record_envelope.record
url = self.get_ingest_endpoint(mce)
raw_mce_obj = mce.proposedSnapshot.to_obj()
mce_obj = _rest_li_ify(raw_mce_obj)
snapshot = {"snapshot": mce_obj}
try:
response = requests.post(url, headers=headers, json=snapshot)
# with open('data.json', 'w') as outfile:
# json.dump(serialized_snapshot, outfile)
response.raise_for_status()
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
except HTTPError as e:
info = response.json()
self.report.report_failure({"e": e, "info": info})
write_callback.on_failure(record_envelope, e, info)
except Exception as e:
self.report.report_failure({"e": e})
write_callback.on_failure(record_envelope, e, {})
def write_record_async(
self,
record_envelope: RecordEnvelope[MetadataChangeEvent],
write_callback: WriteCallback,
) -> None:
mce = record_envelope.record
try:
self.emitter.emit_mce(mce)
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
except OperationalError as e:
self.report.report_failure({"error": e.message, "info": e.info})
write_callback.on_failure(record_envelope, e, e.info)
except Exception as e:
self.report.report_failure({"e": e})
write_callback.on_failure(record_envelope, e, {})
def write_record_async(
self,
record_envelope: RecordEnvelope[Union[MetadataChangeEvent,
MetadataChangeProposal,
MetadataChangeProposalWrapper,
UsageAggregation, ]],
write_callback: WriteCallback,
) -> None:
record = record_envelope.record
try:
self.emitter.emit(record)
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
except OperationalError as e:
# only OperationalErrors should be ignored
if not self.treat_errors_as_warnings:
self.report.report_failure({
"error": e.message,
"info": e.info
})
else:
# trim exception stacktraces when reporting warnings
if "stackTrace" in e.info:
try:
e.info["stackTrace"] = "\n".join(
e.info["stackTrace"].split("\n")[0:2])
except Exception:
# ignore failures in trimming
pass
if isinstance(record, MetadataChangeProposalWrapper):
# include information about the entity that failed
entity_id = cast(MetadataChangeProposalWrapper,
record).entityUrn
e.info["id"] = entity_id
else:
entity_id = None
self.report.report_warning({
"warning": e.message,
"info": e.info
})
write_callback.on_failure(record_envelope, e, e.info)
except Exception as e:
self.report.report_failure({"e": e})
write_callback.on_failure(record_envelope, e, {})
def write_record_async(
self,
record_envelope: RecordEnvelope[
Union[
MetadataChangeEvent,
MetadataChangeProposal,
MetadataChangeProposalWrapper,
UsageAggregation,
]
],
write_callback: WriteCallback,
) -> None:
record = record_envelope.record
try:
self.emitter.emit(record)
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
except OperationalError as e:
self.report.report_failure({"error": e.message, "info": e.info})
write_callback.on_failure(record_envelope, e, e.info)
except Exception as e:
self.report.report_failure({"e": e})
write_callback.on_failure(record_envelope, e, {})
def write_record_async(self, record_envelope: RecordEnvelope,
write_callback: WriteCallback):
print(f'{self.ctx.run_id}:{record_envelope}')
if write_callback:
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
def write_record_async(self, record_envelope: RecordEnvelope,
write_callback: WriteCallback) -> None:
print(f"{record_envelope}")
if write_callback:
self.report.report_record_written(record_envelope)
write_callback.on_success(record_envelope, {})
def write_record_async(self, record_envelope: RecordEnvelope,
callback: WriteCallback) -> None:
self.sink_report.report_record_written(record_envelope)
callback.on_success(record_envelope, {})