def extract(rcol: monetdbe_column,
r: int,
text_factory: Optional[Callable[[str], Any]] = None):
"""
Extracts values from a monetdbe_column.
The text_factory is optional, and wraps the value with a custom user supplied text function.
"""
type_info = monet_c_type_map[rcol.type]
col = ffi.cast(f"monetdbe_column_{type_info.c_string_type} *", rcol)
if col.is_null(col.data + r):
return None
else:
col_data = col.data[r]
if rcol.sql_type.name != ffi.NULL and ffi.string(
rcol.sql_type.name).decode() == 'decimal':
col_data = Decimal(col_data) / (Decimal(10)**
rcol.sql_type.scale)
if type_info.py_converter:
result = type_info.py_converter(col_data)
if rcol.type == lib.monetdbe_str and text_factory:
return text_factory(result)
return result
return col_data
def append(self, table: str, data: Mapping[str, np.ndarray], schema: str = 'sys') -> None:
"""
Directly append an array structure
"""
n_columns = len(data)
existing_columns = list(self.get_columns(schema=schema, table=table))
existing_names, existing_types = zip(*existing_columns)
if not set(existing_names) == set(data.keys()):
error = f"Appended column names ({', '.join(str(i) for i in data.keys())}) " \
f"don't match existing column names ({', '.join(existing_names)})"
raise exceptions.ProgrammingError(error)
work_columns = ffi.new(f'monetdbe_column * [{n_columns}]')
work_objs = []
for column_num, (column_name, existing_type) in enumerate(existing_columns):
column_values = data[column_name]
work_column = ffi.new('monetdbe_column *')
work_type_string, work_type = numpy_monetdb_map(column_values.dtype)
if not work_type == existing_type:
existing_type_string = monet_numpy_map[existing_type][0]
error = f"Type '{work_type_string}' for appended column '{column_name}' " \
f"does not match table type '{existing_type_string}'"
raise exceptions.ProgrammingError(error)
work_column.type = work_type
work_column.count = column_values.shape[0]
work_column.name = ffi.new('char[]', column_name.encode())
work_column.data = ffi.cast(f"{work_type_string} *", ffi.from_buffer(column_values))
work_columns[column_num] = work_column
work_objs.append(work_column)
check_error(lib.monetdbe_append(self._connection, schema.encode(), table.encode(), work_columns, n_columns))
def extract(rcol: monetdbe_column,
r: int,
text_factory: Optional[Callable[[str], Any]] = None):
"""
Extracts values from a monetdbe_column.
The text_factory is optional, and wraps the value with a custom user supplied text function.
"""
type_info = monet_c_type_map[rcol.type]
col = ffi.cast(f"monetdbe_column_{type_info.c_string_type} *", rcol)
if col.is_null(col.data + r):
return None
else:
if type_info.py_converter:
result = type_info.py_converter(col.data[r])
if rcol.type == lib.monetdbe_str and text_factory:
return text_factory(result)
return result
return col.data[r]
def extract(rcol, r: int, text_factory: Optional[Callable[[str], Any]] = None):
"""
Extracts values from a monetdbe_column.
The text_factory is optional, and wraps the value with a custom user supplied text function.
"""
cast_string, cast_function, numpy_type, monetdbe_null = type_map[rcol.type]
col = ffi.cast(f"monetdbe_column_{cast_string} *", rcol)
if col.is_null(col.data[r]):
return None
else:
if cast_function:
result = cast_function(col.data[r])
if rcol.type == lib.monetdbe_str and text_factory:
return text_factory(result)
else:
return result
else:
return col.data[r]
def get_null_value(rcol: monetdbe_column):
type_info = monet_c_type_map[rcol.type]
col = ffi.cast(f"monetdbe_column_{type_info.c_string_type} *", rcol)
return col.null_value
def append(self,
table: str,
data: Mapping[str, np.ndarray],
schema: str = 'sys') -> None:
"""
Directly append an array structure
"""
self._switch()
n_columns = len(data)
existing_columns = list(self.get_columns(schema=schema, table=table))
existing_names, existing_types = zip(*existing_columns)
if not set(existing_names) == set(data.keys()):
error = f"Appended column names ({', '.join(str(i) for i in data.keys())}) " \
f"don't match existing column names ({', '.join(existing_names)})"
raise exceptions.ProgrammingError(error)
work_columns = ffi.new(f'monetdbe_column * [{n_columns}]')
work_objs = []
# cffi_objects assists to keep all in-memory native data structure alive during the execution of this call
cffi_objects = list()
for column_num, (column_name,
existing_type) in enumerate(existing_columns):
column_values = data[column_name]
work_column = ffi.new('monetdbe_column *')
type_info = numpy_monetdb_map(column_values.dtype)
# try to convert the values if types don't match
if type_info.c_type != existing_type:
if type_info.c_type == lib.monetdbe_timestamp and existing_type == lib.monetdbe_date and np.issubdtype(
column_values.dtype, np.datetime64):
"""
We are going to cast to a monetdbe_date and
consider monetdbe_timestamp as a 'base type' to signal this.
"""
type_info = timestamp_to_date()
else:
precision_warning(type_info.c_type, existing_type)
to_numpy_type = monet_c_type_map[existing_type].numpy_type
try:
column_values = column_values.astype(to_numpy_type)
type_info = numpy_monetdb_map(column_values.dtype)
except Exception as e:
existing_type_string = monet_c_type_map[
existing_type].c_string_type
error = f"Can't convert '{type_info.c_string_type}' " \
f"to type '{existing_type_string}' for column '{column_name}': {e} "
raise ValueError(error)
work_column.type = type_info.c_type
work_column.count = column_values.shape[0]
work_column.name = ffi.new('char[]', column_name.encode())
if type_info.numpy_type.kind == 'M':
t = ffi.new('monetdbe_data_timestamp[]', work_column.count)
cffi_objects.append(t)
unit = np.datetime_data(column_values.dtype)[0].encode()
p = ffi.from_buffer("int64_t*", column_values)
lib.initialize_timestamp_array_from_numpy(
self._monetdbe_database, t, work_column.count, p, unit,
existing_type)
work_column.data = t
elif type_info.numpy_type.kind == 'U':
# first massage the numpy array of unicode into a matrix of null terminated rows of bytes.
m = ffi.from_buffer(
"bool*", column_values.mask) if np.ma.isMaskedArray(
column_values) else 0 # type: ignore[attr-defined]
cffi_objects.append(m)
v = np.char.encode(column_values).view('b').reshape(
(work_column.count, -1))
v = np.c_[v, np.zeros(work_column.count, dtype=np.int8)]
stride_length = v.shape[1]
cffi_objects.append(v)
t = ffi.new('char*[]', work_column.count)
cffi_objects.append(t)
p = ffi.from_buffer("char*", v)
cffi_objects.append(p)
lib.initialize_string_array_from_numpy(t, work_column.count, p,
stride_length,
ffi.cast("bool*", m))
work_column.data = t
else:
p = ffi.from_buffer(f"{type_info.c_string_type}*",
column_values)
cffi_objects.append(p)
work_column.data = p
work_columns[column_num] = work_column
work_objs.append(work_column)
check_error(
lib.monetdbe_append(self._monetdbe_database, schema.encode(),
table.encode(), work_columns, n_columns))