class IntervalVariableType(VariableType):
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCIInterval)
#self.initialize_proc = None
#self.finalize_proc = None
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = None
self.is_null_proc = None
#self.set_value_proc = None
#self.get_value_proc = None
self.get_buffer_size_proc = None
self.python_type = INTERVAL
self.oracle_type = oci.SQLT_INTERVAL_DS
self.charset_form = oci.SQLCS_IMPLICIT
self.size = ctypes.sizeof(self.oci_type)
self.is_character_data = False
self.is_variable_length = False
self.can_be_copied = True
self.can_be_in_array = True
self.descriptor_manager = DescriptorManager()
def initialize_proc(self, var, cursor):
self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_INTERVAL_DS, "IntervalVar_Initialize()")
def finalize_proc(self, var):
self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_INTERVAL_DS)
def set_value_proc(self, var, pos, value):
typed_data = self.get_typed_data(var)
if not isinstance(value, timedelta):
raise TypeError("expecting timedelta data")
hours, seconds = divmod(value.seconds, 3600)
minutes, seconds = divmod(seconds, 60)
status = oci.OCIIntervalSetDaySecond(var.environment.handle, var.environment.error_handle,
value.days, hours, minutes, seconds, value.microseconds,
typed_data[pos])
var.environment.check_for_error(status, "IntervalVar_SetValue()")
def get_value_proc(self, var, pos):
typed_data = self.get_typed_data(var)
return oracle_interval_to_python_delta(var.environment, typed_data[pos])
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCIInterval)
#self.initialize_proc = None
#self.finalize_proc = None
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = None
self.is_null_proc = None
#self.set_value_proc = None
#self.get_value_proc = None
self.get_buffer_size_proc = None
self.python_type = INTERVAL
self.oracle_type = oci.SQLT_INTERVAL_DS
self.charset_form = oci.SQLCS_IMPLICIT
self.size = ctypes.sizeof(self.oci_type)
self.is_character_data = False
self.is_variable_length = False
self.can_be_copied = True
self.can_be_in_array = True
self.descriptor_manager = DescriptorManager()
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCILobLocator)
self.initialize_proc = self.initialize
self.finalize_proc = self.finalize
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = self.pre_fetch
self.is_null_proc = None
self.set_value_proc = self.set_value
self.get_value_proc = self.get_value
self.get_buffer_size_proc = None
#self.python_type = None
#self.oracle_type = None
#self.charset_form = None
self.size = ctypes.sizeof(self.oci_type)
#self.is_character_data = None
self.is_variable_length = False
self.can_be_copied = False
self.can_be_in_array = False
self.descriptor_manager = DescriptorManager()
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCILobLocator)
self.initialize_proc = self.initialize
self.finalize_proc = self.finalize
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = self.pre_fetch
self.is_null_proc = None
self.set_value_proc = self.set_value
self.get_value_proc = self.get_value
self.get_buffer_size_proc = None
#self.python_type = None
#self.oracle_type = None
#self.charset_form = None
self.size = ctypes.sizeof(self.oci_type)
#self.is_character_data = None
self.is_variable_length = False
self.can_be_copied = False
self.can_be_in_array = False
self.descriptor_manager = DescriptorManager()
class BaseLobVariableType(VariableType):
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCILobLocator)
self.initialize_proc = self.initialize
self.finalize_proc = self.finalize
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = self.pre_fetch
self.is_null_proc = None
self.set_value_proc = self.set_value
self.get_value_proc = self.get_value
self.get_buffer_size_proc = None
#self.python_type = None
#self.oracle_type = None
#self.charset_form = None
self.size = ctypes.sizeof(self.oci_type)
#self.is_character_data = None
self.is_variable_length = False
self.can_be_copied = False
self.can_be_in_array = False
self.descriptor_manager = DescriptorManager()
def initialize(self, var, cursor):
var.connection = cursor.connection
self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_LOB, "LobVar_Initialize()")
def finalize(self, var):
self.pre_fetch(var)
self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_LOB)
def pre_fetch(self, var):
"""Free temporary LOBs prior to fetch."""
typed_data = self.get_typed_data(var)
for i in xrange(var.allocelems):
if typed_data[i]:
c_is_temporary = ctypes.c_int()
status = oci.OCILobIsTemporary(var.environment.handle,
var.environment.error_handle, typed_data[i], byref(c_is_temporary))
var.environment.check_for_error(status, "LobVar_PreFetch(): is temporary LOB?")
is_temporary = c_is_temporary.value
# connection can be closed already, therefore we need to check for a non-null handle
if is_temporary and var.connection.handle:
status = oci.OCILobFreeTemporary(var.connection.handle, var.environment.error_handle,
typed_data[i])
var.environment.check_for_error(status, "LobVar_PreFetch(): free temporary LOB")
def set_value(self, var, pos, value):
# make sure have temporary LOBs set up
typed_data = self.get_typed_data(var)
c_is_temporary = ctypes.c_int()
status = oci.OCILobIsTemporary(var.environment.handle, var.environment.error_handle,
typed_data[pos], byref(c_is_temporary))
var.environment.check_for_error(status, "LobVar_SetValue(): is temporary?")
is_temporary = c_is_temporary.value
if not is_temporary:
if var.type.oracle_type == oci.SQLT_BLOB:
lob_type = oci.OCI_TEMP_BLOB
else:
lob_type = oci.OCI_TEMP_CLOB
status = oci.OCILobCreateTemporary(var.connection.handle,
var.environment.error_handle, typed_data[pos],
oci.OCI_DEFAULT, var.type.charset_form, lob_type, 0,
oci.OCI_DURATION_SESSION)
var.environment.check_for_error(status, "LobVar_SetValue(): create temporary")
# trim the current value
status = oci.OCILobTrim(var.connection.handle, var.environment.error_handle,
typed_data[pos], 0)
var.environment.check_for_error(status, "LobVar_SetValue(): trim")
# set the current value
self._write(var, pos, value, 1)
def _write(self, var, pos, data_obj, offset):
"""Write data to the LOB variable."""
# verify the data type
var_type = var.type
if var_type == vt_BFILE:
raise TypeError("BFILEs are read only")
if var_type == vt_BLOB:
buffer = cxBuffer.new_from_object(data_obj, var.environment.encoding)
amount = buffer.size
elif not python3_or_better() and var_type == vt_NCLOB:
buffer = cxBuffer.new_from_object(data_obj, var.environment.nencoding)
amount = buffer.size
else:
buffer = cxBuffer.new_from_object(data_obj, var.environment.encoding)
if var.environment.fixedWidth and var.environment.maxBytesPerCharacter > 1:
amount = buffer.size / var.environment.maxBytesPerCharacter
else:
amount = buffer.size
# nothing to do if no data to write
if amount == 0:
return amount
typed_data = self.get_typed_data(var)
c_amount = oci.ub4(amount)
callback_lob_write_type = oci.OCILobWrite.argtypes[9]
null_oci_callback = callback_lob_write_type()
status = oci.OCILobWrite(var.connection.handle,
var.environment.error_handle, typed_data[pos], byref(c_amount), offset,
buffer.ptr, buffer.size, oci.OCI_ONE_PIECE, None, null_oci_callback, 0,
var.type.charset_form)
var.environment.check_for_error(status, "LobVar_Write()")
amount = c_amount.value # not sure if the write can change it, i dont think so, but just for correctness
return amount
def get_value(self, var, pos):
return LOB(var, pos)
import os
import numpy as np
import pickle
from descriptor_manager import DescriptorManager
import utilities as utils
if __name__ == '__main__':
wdir = '../../dataset/train-x512-processed-stage2'
idx = os.path.join(wdir, 'idx_test.txt')
manager = DescriptorManager(os.path.join(wdir, 'descriptors'))
print('Reading data...')
cervixes = utils.readCervixes(idx, "test")
print('Building descriptors')
cervix_kmeans_model = None
cervix_kmeans_name = 'models/kmeans_pixels-20000_clusters-64'
with open(cervix_kmeans_name, 'rb') as f:
cervix_kmeans_model = pickle.load(f)
channel_kmeans_model = None
channel_kmeans_name = 'models/kmeans_channel_pixels-20000_clusters-64'
with open(channel_kmeans_name, 'rb') as f:
channel_kmeans_model = pickle.load(f)
configs = [
['RGB-hist', {
'bins': 32,
# ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-512"],
# ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-256"],
['KMeans', {
'kmeans_model': True,
'mask': True
}, "cervix-20000-64"],
# ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-16"],
# ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-50000-64"],
['KMeans', {
'kmeans_model': True,
'mask': True
}, "channel-20000-64"],
]
print('Loading descriptors...')
manager = DescriptorManager(os.path.join(wdir, 'descriptors'))
cervixes_data = []
cervixes_labels = []
for c in cervixes:
cervixes_data.append(manager.buildFeatures(c['code_name'], configs))
cervixes_labels.append(c['type'])
print('Feature length: ' + str(cervixes_data[0].shape))
FOLD_SIZE = 5
BATCH_SIZE = len(cervixes_data) // FOLD_SIZE
WIN_PROB = 0.66
LOS_PROB = (1 - WIN_PROB) / 2
print('Start %d-fold cross-validation' % FOLD_SIZE)
from descriptor_manager import DescriptorManager
import utilities as utils
if (__name__ == '__main__'):
wdir = '../../dataset/train-x512-processed-stage2'
train_idx = os.path.join(wdir, 'idx_all.txt-shuf.txt')
valid_idx = os.path.join(wdir, 'idx_test.txt')
# valid_idx = 'idx-mask.txt-val.txt-processed.txt'
print('Reading data...')
train = utils.readCervixes(train_idx, "cervix", with_imgs=False)
valid = utils.readCervixes(valid_idx, "test", with_imgs=False)
# valid = utils.readCervixes('/home/ar/bitbucket.org/kaggle_intel_image_preprocessing-Dataset/train-x512-processed', valid_idx, "CNN")
manager = DescriptorManager(os.path.join(wdir, 'descriptors'))
configs = [
['RGB-hist', {'bins': 32, 'mask': True}],
['RGB-hist', {'bins': 128, 'mask': True}],
['RGB-hist', {'bins': 256, 'mask': True}],
['LBP', {'radius': 12, 'numPoints': 36, 'mask': True}],
['LBP', {'radius': 16, 'numPoints': 48, 'mask': True}],
['LBP', {'radius': 32, 'numPoints': 160, 'mask': True}],
# ['RGB-hist', {'bins': 32, 'mask': True}, "channel-mask"],
# ['RGB-hist', {'bins': 128, 'mask': True}, "channel-mask"],
# ['RGB-hist', {'bins': 256, 'mask': True}, "channel-mask"],
# ['LBP', {'radius': 12, 'numPoints': 36, 'mask': True}, "channel-mask"],
# ['LBP', {'radius': 16, 'numPoints': 48, 'mask': True}, "channel-mask"],
# ['LBP', {'radius': 32, 'numPoints': 160, 'mask': True}, "channel-mask"],
# ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-512"],
class TimestampType(VariableType):
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCIDateTime)
self.initialize_proc = self.initialize
self.finalize_proc = self.finalize
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = None
self.is_null_proc = None
self.set_value_proc = self.set_value
self.get_value_proc = self.get_value
self.get_buffer_size_proc = None
self.python_type = TIMESTAMP
self.oracle_type = oci.SQLT_TIMESTAMP
self.charset_form = oci.SQLCS_IMPLICIT
self.size = ctypes.sizeof(self.oci_type)
self.is_character_data = False
self.is_variable_length = False
self.can_be_copied = True
self.can_be_in_array = True
self.descriptor_manager = DescriptorManager()
def initialize(self, var, cursor):
self.descriptor_manager.initialize(self, var, cursor,
oci.OCI_DTYPE_TIMESTAMP,
"TimestampVar_Initialize()")
def finalize(self, var):
self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_TIMESTAMP)
def set_value(self, var, pos, value):
# make sure a timestamp is being bound
if not isinstance(value, datetime):
raise TypeError("expecting timestamp data")
typed_data = self.get_typed_data(var)
# store a copy of the value
status = oci.OCIDateTimeConstruct(var.environment.handle,
var.environment.error_handle,
typed_data[pos], value.year,
value.month, value.day, value.hour,
value.minute, value.second,
value.microsecond * 1000, None, 0)
var.environment.check_for_error(
status, "TimestampVar_SetValue(): create structure")
c_invalid = oci.uword()
status = oci.OCIDateTimeCheck(var.environment.handle,
var.environment.error_handle,
typed_data[pos], byref(c_invalid))
invalid = c_invalid.value
var.environment.check_for_error(status, "TimestampVar_SetValue()")
if invalid:
raise DataError("invalid date")
def get_value(self, var, pos):
typed_data = self.get_typed_data(var)
return oracle_timestamp_to_python_date(var.environment,
typed_data[pos])
class BaseLobVariableType(VariableType):
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCILobLocator)
self.initialize_proc = self.initialize
self.finalize_proc = self.finalize
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = self.pre_fetch
self.is_null_proc = None
self.set_value_proc = self.set_value
self.get_value_proc = self.get_value
self.get_buffer_size_proc = None
#self.python_type = None
#self.oracle_type = None
#self.charset_form = None
self.size = ctypes.sizeof(self.oci_type)
#self.is_character_data = None
self.is_variable_length = False
self.can_be_copied = False
self.can_be_in_array = False
self.descriptor_manager = DescriptorManager()
def initialize(self, var, cursor):
var.connection = cursor.connection
self.descriptor_manager.initialize(self, var, cursor,
oci.OCI_DTYPE_LOB,
"LobVar_Initialize()")
def finalize(self, var):
self.pre_fetch(var)
self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_LOB)
def pre_fetch(self, var):
"""Free temporary LOBs prior to fetch."""
typed_data = self.get_typed_data(var)
for i in xrange(var.allocelems):
if typed_data[i]:
c_is_temporary = ctypes.c_int()
status = oci.OCILobIsTemporary(var.environment.handle,
var.environment.error_handle,
typed_data[i],
byref(c_is_temporary))
var.environment.check_for_error(
status, "LobVar_PreFetch(): is temporary LOB?")
is_temporary = c_is_temporary.value
# connection can be closed already, therefore we need to check for a non-null handle
if is_temporary and var.connection.handle:
status = oci.OCILobFreeTemporary(
var.connection.handle, var.environment.error_handle,
typed_data[i])
var.environment.check_for_error(
status, "LobVar_PreFetch(): free temporary LOB")
def set_value(self, var, pos, value):
# make sure have temporary LOBs set up
typed_data = self.get_typed_data(var)
c_is_temporary = ctypes.c_int()
status = oci.OCILobIsTemporary(var.environment.handle,
var.environment.error_handle,
typed_data[pos], byref(c_is_temporary))
var.environment.check_for_error(status,
"LobVar_SetValue(): is temporary?")
is_temporary = c_is_temporary.value
if not is_temporary:
if var.type.oracle_type == oci.SQLT_BLOB:
lob_type = oci.OCI_TEMP_BLOB
else:
lob_type = oci.OCI_TEMP_CLOB
status = oci.OCILobCreateTemporary(var.connection.handle,
var.environment.error_handle,
typed_data[pos],
oci.OCI_DEFAULT,
var.type.charset_form, lob_type,
0, oci.OCI_DURATION_SESSION)
var.environment.check_for_error(
status, "LobVar_SetValue(): create temporary")
# trim the current value
status = oci.OCILobTrim(var.connection.handle,
var.environment.error_handle, typed_data[pos],
0)
var.environment.check_for_error(status, "LobVar_SetValue(): trim")
# set the current value
self._write(var, pos, value, 1)
def _write(self, var, pos, data_obj, offset):
"""Write data to the LOB variable."""
# verify the data type
var_type = var.type
if var_type == vt_BFILE:
raise TypeError("BFILEs are read only")
if var_type == vt_BLOB:
buffer = cxBuffer.new_from_object(data_obj,
var.environment.encoding)
amount = buffer.size
elif not python3_or_better() and var_type == vt_NCLOB:
buffer = cxBuffer.new_from_object(data_obj,
var.environment.nencoding)
amount = buffer.size
else:
buffer = cxBuffer.new_from_object(data_obj,
var.environment.encoding)
if var.environment.fixedWidth and var.environment.maxBytesPerCharacter > 1:
amount = buffer.size / var.environment.maxBytesPerCharacter
else:
amount = buffer.size
# nothing to do if no data to write
if amount == 0:
return amount
typed_data = self.get_typed_data(var)
c_amount = oci.ub4(amount)
callback_lob_write_type = oci.OCILobWrite.argtypes[9]
null_oci_callback = callback_lob_write_type()
status = oci.OCILobWrite(var.connection.handle,
var.environment.error_handle, typed_data[pos],
byref(c_amount), offset, buffer.ptr,
buffer.size, oci.OCI_ONE_PIECE, None,
null_oci_callback, 0, var.type.charset_form)
var.environment.check_for_error(status, "LobVar_Write()")
amount = c_amount.value # not sure if the write can change it, i dont think so, but just for correctness
return amount
def get_value(self, var, pos):
return LOB(var, pos)
class TimestampType(VariableType):
def __init__(self):
VariableType.__init__(self)
self.oci_type = oci.POINTER(oci.OCIDateTime)
self.initialize_proc = self.initialize
self.finalize_proc = self.finalize
self.pre_define_proc = None
self.post_define_proc = None
self.pre_fetch_proc = None
self.is_null_proc = None
self.set_value_proc = self.set_value
self.get_value_proc = self.get_value
self.get_buffer_size_proc = None
self.python_type = TIMESTAMP
self.oracle_type = oci.SQLT_TIMESTAMP
self.charset_form = oci.SQLCS_IMPLICIT
self.size = ctypes.sizeof(self.oci_type)
self.is_character_data = False
self.is_variable_length = False
self.can_be_copied = True
self.can_be_in_array = True
self.descriptor_manager = DescriptorManager()
def initialize(self, var, cursor):
self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_TIMESTAMP, "TimestampVar_Initialize()")
def finalize(self, var):
self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_TIMESTAMP)
def set_value(self, var, pos, value):
# make sure a timestamp is being bound
if not isinstance(value, datetime):
raise TypeError("expecting timestamp data")
typed_data = self.get_typed_data(var)
# store a copy of the value
status = oci.OCIDateTimeConstruct(var.environment.handle,
var.environment.error_handle, typed_data[pos],
value.year,
value.month,
value.day,
value.hour,
value.minute,
value.second,
value.microsecond * 1000, None, 0);
var.environment.check_for_error(status, "TimestampVar_SetValue(): create structure")
c_invalid = oci.uword()
status = oci.OCIDateTimeCheck(var.environment.handle,
var.environment.error_handle, typed_data[pos], byref(c_invalid))
invalid = c_invalid.value
var.environment.check_for_error(status, "TimestampVar_SetValue()")
if invalid:
raise DataError("invalid date")
def get_value(self, var, pos):
typed_data = self.get_typed_data(var)
return oracle_timestamp_to_python_date(var.environment, typed_data[pos])
