def add_citation(
self,
comment: str,
batch_request: Optional[
Union[str, Dict[str, Union[str, Dict[str, Any]]]]
] = None,
batch_definition: Optional[dict] = None,
batch_spec: Optional[dict] = None,
batch_kwargs: Optional[dict] = None,
batch_markers: Optional[dict] = None,
batch_parameters: Optional[dict] = None,
citation_date: Optional[Union[str, datetime.datetime]] = None,
):
if "citations" not in self.meta:
self.meta["citations"] = []
if isinstance(citation_date, str):
citation_date = parse_string_to_datetime(datetime_string=citation_date)
citation_date = citation_date or datetime.datetime.now(datetime.timezone.utc)
self.meta["citations"].append(
{
"citation_date": get_datetime_string_from_strftime_format(
format_str="%Y-%m-%dT%H:%M:%S.%fZ", datetime_obj=citation_date
),
"batch_request": batch_request,
"batch_definition": batch_definition,
"batch_spec": batch_spec,
"batch_kwargs": batch_kwargs,
"batch_markers": batch_markers,
"batch_parameters": batch_parameters,
"comment": comment,
}
)
def get_batch_key(self, batch_definition: BatchDefinition) -> Any:
batch_identifiers: dict = batch_definition.batch_identifiers
partition_value: Any = batch_identifiers[self.name]
dt: datetime.date = parse_string_to_datetime(
datetime_string=partition_value,
datetime_format_string=self._datetime_format,
)
return datetime_to_int(dt=dt)
def get_partition_key(self, batch_definition: BatchDefinition) -> Any:
partition_definition: dict = batch_definition.partition_definition
partition_value: Any = partition_definition[self.name]
dt: datetime.date = parse_string_to_datetime(
datetime_string=partition_value,
datetime_format_string=self._datetime_format,
)
return datetime_to_int(dt=dt)
def add_citation(
self,
comment: str,
batch_request: Optional[Union[str, Dict[str,
Union[str,
Dict[str,
Any]]]]] = None,
batch_definition: Optional[dict] = None,
batch_spec: Optional[dict] = None,
batch_kwargs: Optional[dict] = None,
batch_markers: Optional[dict] = None,
batch_parameters: Optional[dict] = None,
profiler_config: Optional[dict] = None,
citation_date: Optional[Union[str, datetime.datetime]] = None,
):
if "citations" not in self.meta:
self.meta["citations"] = []
if isinstance(citation_date, str):
citation_date = parse_string_to_datetime(
datetime_string=citation_date)
citation_date = citation_date or datetime.datetime.now(
datetime.timezone.utc)
citation: Dict[str,
Any] = {
"citation_date":
get_datetime_string_from_strftime_format(
format_str="%Y-%m-%dT%H:%M:%S.%fZ",
datetime_obj=citation_date),
"batch_request":
batch_request,
"batch_definition":
batch_definition,
"batch_spec":
batch_spec,
"batch_kwargs":
batch_kwargs,
"batch_markers":
batch_markers,
"batch_parameters":
batch_parameters,
"profiler_config":
profiler_config,
"comment":
comment,
}
ge.util.filter_properties_dict(properties=citation,
clean_falsy=True,
inplace=True)
self.meta["citations"].append(citation)
