def add_citation( self, comment: str, batch_request: Optional[ Union[str, Dict[str, Union[str, Dict[str, Any]]]] ] = None, batch_definition: Optional[dict] = None, batch_spec: Optional[dict] = None, batch_kwargs: Optional[dict] = None, batch_markers: Optional[dict] = None, batch_parameters: Optional[dict] = None, citation_date: Optional[Union[str, datetime.datetime]] = None, ): if "citations" not in self.meta: self.meta["citations"] = [] if isinstance(citation_date, str): citation_date = parse_string_to_datetime(datetime_string=citation_date) citation_date = citation_date or datetime.datetime.now(datetime.timezone.utc) self.meta["citations"].append( { "citation_date": get_datetime_string_from_strftime_format( format_str="%Y-%m-%dT%H:%M:%S.%fZ", datetime_obj=citation_date ), "batch_request": batch_request, "batch_definition": batch_definition, "batch_spec": batch_spec, "batch_kwargs": batch_kwargs, "batch_markers": batch_markers, "batch_parameters": batch_parameters, "comment": comment, } )
def get_batch_key(self, batch_definition: BatchDefinition) -> Any: batch_identifiers: dict = batch_definition.batch_identifiers partition_value: Any = batch_identifiers[self.name] dt: datetime.date = parse_string_to_datetime( datetime_string=partition_value, datetime_format_string=self._datetime_format, ) return datetime_to_int(dt=dt)
def get_partition_key(self, batch_definition: BatchDefinition) -> Any: partition_definition: dict = batch_definition.partition_definition partition_value: Any = partition_definition[self.name] dt: datetime.date = parse_string_to_datetime( datetime_string=partition_value, datetime_format_string=self._datetime_format, ) return datetime_to_int(dt=dt)
def add_citation( self, comment: str, batch_request: Optional[Union[str, Dict[str, Union[str, Dict[str, Any]]]]] = None, batch_definition: Optional[dict] = None, batch_spec: Optional[dict] = None, batch_kwargs: Optional[dict] = None, batch_markers: Optional[dict] = None, batch_parameters: Optional[dict] = None, profiler_config: Optional[dict] = None, citation_date: Optional[Union[str, datetime.datetime]] = None, ): if "citations" not in self.meta: self.meta["citations"] = [] if isinstance(citation_date, str): citation_date = parse_string_to_datetime( datetime_string=citation_date) citation_date = citation_date or datetime.datetime.now( datetime.timezone.utc) citation: Dict[str, Any] = { "citation_date": get_datetime_string_from_strftime_format( format_str="%Y-%m-%dT%H:%M:%S.%fZ", datetime_obj=citation_date), "batch_request": batch_request, "batch_definition": batch_definition, "batch_spec": batch_spec, "batch_kwargs": batch_kwargs, "batch_markers": batch_markers, "batch_parameters": batch_parameters, "profiler_config": profiler_config, "comment": comment, } ge.util.filter_properties_dict(properties=citation, clean_falsy=True, inplace=True) self.meta["citations"].append(citation)