def _create_columns(self, header_df):
from weld.weldobject import WeldObject
columns = {}
for column_name in header_df:
data_id = LazyResult.generate_data_id(column_name)
column = Column(column_name, self, data_id, header_df[column_name].dtype)
columns[column_name] = column
weld_input_name = WeldObject.generate_input_name(data_id)
LazyResult.register_lazy_data(weld_input_name, column)
return columns
def _create_variables(self):
variables = OrderedDict()
for kv in self.ds.variables.items():
# generate a data_id to act as placeholder to the data
data_id = LazyResult.generate_data_id(kv[0])
weld_obj, weld_input_id = LazyResult.generate_placeholder_weld_object(
data_id, Variable.encoder, Variable.decoder)
variable = Variable(self.file_id, kv[0], kv[1].dimensions,
kv[1].shape, kv[1].__dict__, weld_obj,
kv[1].dtype)
LazyResult.register_lazy_data(weld_input_id, variable)
variables[kv[0]] = variable
return variables
def _create_columns(self, header_df):
from weld.weldobject import WeldObject
columns = {}
for column_name in header_df:
data_id = LazyResult.generate_data_id(column_name)
column = Column(column_name, self, data_id,
header_df[column_name].dtype)
weld_input_name = WeldObject.generate_input_name(data_id)
LazyResult.register_lazy_data(weld_input_name, column)
# force read it eagerly
LazyResult.input_mapping[str(
weld_input_name)] = column.eager_read()
columns[column_name] = column
return columns
