def load(self, value_type, **kwargs):
cache_key = TargetCacheKey(target=self, value_type=value_type)
if cache_key in TARGET_CACHE:
logger.info("Using cached data value for target='%s'", self)
return TARGET_CACHE[cache_key]
m = get_marshaller_ctrl(self, value_type)
value = m.load(**kwargs)
TARGET_CACHE[cache_key] = value
return value
def dump(self, value, value_type=None, **kwargs):
from targets.values import get_value_type_of_obj, get_value_type_of_type
from targets.values import ObjectValueType
if value_type:
value_type = get_value_type_of_type(value_type)
else:
value_type = get_value_type_of_obj(value, ObjectValueType())
try:
m = get_marshaller_ctrl(self, value_type=value_type)
m.dump(value, **kwargs)
except Exception as ex:
raise friendly_error.failed_to_write_task_output(
ex, self, value_type=value_type
)
cache_key = TargetCacheKey(target=self, value_type=value_type)
TARGET_CACHE[cache_key] = value
def to(self, df, config=None, **kwargs):
pd_m = get_marshaller_ctrl(self.target, value_type=DataFrame, config=config)
return pd_m.dump(df, **kwargs)
def read_partitioned(self, config=None, **kwargs):
pd_m = get_marshaller_ctrl(self.target, DataFrame, config=config)
for t in pd_m.load_partitioned(**kwargs):
yield t
def read(self, config=None, **kwargs):
# type: (file, **Any) -> 'DataFrame'
pd_m = get_marshaller_ctrl(self.target, DataFrame, config=config)
return pd_m.load(**kwargs)
def _dump(self, value_type, config, value, **kwargs):
marshaler_ctrl = get_marshaller_ctrl(target=self.target,
value_type=value_type,
config=config)
return marshaler_ctrl.dump(value, **kwargs)
def _load(self, value_type, config, **kwargs):
marshaler_ctrl = get_marshaller_ctrl(target=self.target,
value_type=value_type,
config=config)
return marshaler_ctrl.load(**kwargs)