def load(self, value_type, **kwargs): cache_key = TargetCacheKey(target=self, value_type=value_type) if cache_key in TARGET_CACHE: logger.info("Using cached data value for target='%s'", self) return TARGET_CACHE[cache_key] m = get_marshaller_ctrl(self, value_type) value = m.load(**kwargs) TARGET_CACHE[cache_key] = value return value
def dump(self, value, value_type=None, **kwargs): from targets.values import get_value_type_of_obj, get_value_type_of_type from targets.values import ObjectValueType if value_type: value_type = get_value_type_of_type(value_type) else: value_type = get_value_type_of_obj(value, ObjectValueType()) try: m = get_marshaller_ctrl(self, value_type=value_type) m.dump(value, **kwargs) except Exception as ex: raise friendly_error.failed_to_write_task_output( ex, self, value_type=value_type ) cache_key = TargetCacheKey(target=self, value_type=value_type) TARGET_CACHE[cache_key] = value
def to(self, df, config=None, **kwargs): pd_m = get_marshaller_ctrl(self.target, value_type=DataFrame, config=config) return pd_m.dump(df, **kwargs)
def read_partitioned(self, config=None, **kwargs): pd_m = get_marshaller_ctrl(self.target, DataFrame, config=config) for t in pd_m.load_partitioned(**kwargs): yield t
def read(self, config=None, **kwargs): # type: (file, **Any) -> 'DataFrame' pd_m = get_marshaller_ctrl(self.target, DataFrame, config=config) return pd_m.load(**kwargs)
def _dump(self, value_type, config, value, **kwargs): marshaler_ctrl = get_marshaller_ctrl(target=self.target, value_type=value_type, config=config) return marshaler_ctrl.dump(value, **kwargs)
def _load(self, value_type, config, **kwargs): marshaler_ctrl = get_marshaller_ctrl(target=self.target, value_type=value_type, config=config) return marshaler_ctrl.load(**kwargs)