def _create_columns(self, header_df): from weld.weldobject import WeldObject columns = {} for column_name in header_df: data_id = LazyResult.generate_data_id(column_name) column = Column(column_name, self, data_id, header_df[column_name].dtype) columns[column_name] = column weld_input_name = WeldObject.generate_input_name(data_id) LazyResult.register_lazy_data(weld_input_name, column) return columns
def _create_variables(self): variables = OrderedDict() for kv in self.ds.variables.items(): # generate a data_id to act as placeholder to the data data_id = LazyResult.generate_data_id(kv[0]) weld_obj, weld_input_id = LazyResult.generate_placeholder_weld_object( data_id, Variable.encoder, Variable.decoder) variable = Variable(self.file_id, kv[0], kv[1].dimensions, kv[1].shape, kv[1].__dict__, weld_obj, kv[1].dtype) LazyResult.register_lazy_data(weld_input_id, variable) variables[kv[0]] = variable return variables
def _create_columns(self, header_df): from weld.weldobject import WeldObject columns = {} for column_name in header_df: data_id = LazyResult.generate_data_id(column_name) column = Column(column_name, self, data_id, header_df[column_name].dtype) weld_input_name = WeldObject.generate_input_name(data_id) LazyResult.register_lazy_data(weld_input_name, column) # force read it eagerly LazyResult.input_mapping[str( weld_input_name)] = column.eager_read() columns[column_name] = column return columns