class IntervalVariableType(VariableType): def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCIInterval) #self.initialize_proc = None #self.finalize_proc = None self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = None self.is_null_proc = None #self.set_value_proc = None #self.get_value_proc = None self.get_buffer_size_proc = None self.python_type = INTERVAL self.oracle_type = oci.SQLT_INTERVAL_DS self.charset_form = oci.SQLCS_IMPLICIT self.size = ctypes.sizeof(self.oci_type) self.is_character_data = False self.is_variable_length = False self.can_be_copied = True self.can_be_in_array = True self.descriptor_manager = DescriptorManager() def initialize_proc(self, var, cursor): self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_INTERVAL_DS, "IntervalVar_Initialize()") def finalize_proc(self, var): self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_INTERVAL_DS) def set_value_proc(self, var, pos, value): typed_data = self.get_typed_data(var) if not isinstance(value, timedelta): raise TypeError("expecting timedelta data") hours, seconds = divmod(value.seconds, 3600) minutes, seconds = divmod(seconds, 60) status = oci.OCIIntervalSetDaySecond(var.environment.handle, var.environment.error_handle, value.days, hours, minutes, seconds, value.microseconds, typed_data[pos]) var.environment.check_for_error(status, "IntervalVar_SetValue()") def get_value_proc(self, var, pos): typed_data = self.get_typed_data(var) return oracle_interval_to_python_delta(var.environment, typed_data[pos])
def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCIInterval) #self.initialize_proc = None #self.finalize_proc = None self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = None self.is_null_proc = None #self.set_value_proc = None #self.get_value_proc = None self.get_buffer_size_proc = None self.python_type = INTERVAL self.oracle_type = oci.SQLT_INTERVAL_DS self.charset_form = oci.SQLCS_IMPLICIT self.size = ctypes.sizeof(self.oci_type) self.is_character_data = False self.is_variable_length = False self.can_be_copied = True self.can_be_in_array = True self.descriptor_manager = DescriptorManager()
def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCILobLocator) self.initialize_proc = self.initialize self.finalize_proc = self.finalize self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = self.pre_fetch self.is_null_proc = None self.set_value_proc = self.set_value self.get_value_proc = self.get_value self.get_buffer_size_proc = None #self.python_type = None #self.oracle_type = None #self.charset_form = None self.size = ctypes.sizeof(self.oci_type) #self.is_character_data = None self.is_variable_length = False self.can_be_copied = False self.can_be_in_array = False self.descriptor_manager = DescriptorManager()
class BaseLobVariableType(VariableType): def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCILobLocator) self.initialize_proc = self.initialize self.finalize_proc = self.finalize self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = self.pre_fetch self.is_null_proc = None self.set_value_proc = self.set_value self.get_value_proc = self.get_value self.get_buffer_size_proc = None #self.python_type = None #self.oracle_type = None #self.charset_form = None self.size = ctypes.sizeof(self.oci_type) #self.is_character_data = None self.is_variable_length = False self.can_be_copied = False self.can_be_in_array = False self.descriptor_manager = DescriptorManager() def initialize(self, var, cursor): var.connection = cursor.connection self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_LOB, "LobVar_Initialize()") def finalize(self, var): self.pre_fetch(var) self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_LOB) def pre_fetch(self, var): """Free temporary LOBs prior to fetch.""" typed_data = self.get_typed_data(var) for i in xrange(var.allocelems): if typed_data[i]: c_is_temporary = ctypes.c_int() status = oci.OCILobIsTemporary(var.environment.handle, var.environment.error_handle, typed_data[i], byref(c_is_temporary)) var.environment.check_for_error(status, "LobVar_PreFetch(): is temporary LOB?") is_temporary = c_is_temporary.value # connection can be closed already, therefore we need to check for a non-null handle if is_temporary and var.connection.handle: status = oci.OCILobFreeTemporary(var.connection.handle, var.environment.error_handle, typed_data[i]) var.environment.check_for_error(status, "LobVar_PreFetch(): free temporary LOB") def set_value(self, var, pos, value): # make sure have temporary LOBs set up typed_data = self.get_typed_data(var) c_is_temporary = ctypes.c_int() status = oci.OCILobIsTemporary(var.environment.handle, var.environment.error_handle, typed_data[pos], byref(c_is_temporary)) var.environment.check_for_error(status, "LobVar_SetValue(): is temporary?") is_temporary = c_is_temporary.value if not is_temporary: if var.type.oracle_type == oci.SQLT_BLOB: lob_type = oci.OCI_TEMP_BLOB else: lob_type = oci.OCI_TEMP_CLOB status = oci.OCILobCreateTemporary(var.connection.handle, var.environment.error_handle, typed_data[pos], oci.OCI_DEFAULT, var.type.charset_form, lob_type, 0, oci.OCI_DURATION_SESSION) var.environment.check_for_error(status, "LobVar_SetValue(): create temporary") # trim the current value status = oci.OCILobTrim(var.connection.handle, var.environment.error_handle, typed_data[pos], 0) var.environment.check_for_error(status, "LobVar_SetValue(): trim") # set the current value self._write(var, pos, value, 1) def _write(self, var, pos, data_obj, offset): """Write data to the LOB variable.""" # verify the data type var_type = var.type if var_type == vt_BFILE: raise TypeError("BFILEs are read only") if var_type == vt_BLOB: buffer = cxBuffer.new_from_object(data_obj, var.environment.encoding) amount = buffer.size elif not python3_or_better() and var_type == vt_NCLOB: buffer = cxBuffer.new_from_object(data_obj, var.environment.nencoding) amount = buffer.size else: buffer = cxBuffer.new_from_object(data_obj, var.environment.encoding) if var.environment.fixedWidth and var.environment.maxBytesPerCharacter > 1: amount = buffer.size / var.environment.maxBytesPerCharacter else: amount = buffer.size # nothing to do if no data to write if amount == 0: return amount typed_data = self.get_typed_data(var) c_amount = oci.ub4(amount) callback_lob_write_type = oci.OCILobWrite.argtypes[9] null_oci_callback = callback_lob_write_type() status = oci.OCILobWrite(var.connection.handle, var.environment.error_handle, typed_data[pos], byref(c_amount), offset, buffer.ptr, buffer.size, oci.OCI_ONE_PIECE, None, null_oci_callback, 0, var.type.charset_form) var.environment.check_for_error(status, "LobVar_Write()") amount = c_amount.value # not sure if the write can change it, i dont think so, but just for correctness return amount def get_value(self, var, pos): return LOB(var, pos)
import os import numpy as np import pickle from descriptor_manager import DescriptorManager import utilities as utils if __name__ == '__main__': wdir = '../../dataset/train-x512-processed-stage2' idx = os.path.join(wdir, 'idx_test.txt') manager = DescriptorManager(os.path.join(wdir, 'descriptors')) print('Reading data...') cervixes = utils.readCervixes(idx, "test") print('Building descriptors') cervix_kmeans_model = None cervix_kmeans_name = 'models/kmeans_pixels-20000_clusters-64' with open(cervix_kmeans_name, 'rb') as f: cervix_kmeans_model = pickle.load(f) channel_kmeans_model = None channel_kmeans_name = 'models/kmeans_channel_pixels-20000_clusters-64' with open(channel_kmeans_name, 'rb') as f: channel_kmeans_model = pickle.load(f) configs = [ ['RGB-hist', { 'bins': 32,
# ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-512"], # ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-256"], ['KMeans', { 'kmeans_model': True, 'mask': True }, "cervix-20000-64"], # ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-16"], # ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-50000-64"], ['KMeans', { 'kmeans_model': True, 'mask': True }, "channel-20000-64"], ] print('Loading descriptors...') manager = DescriptorManager(os.path.join(wdir, 'descriptors')) cervixes_data = [] cervixes_labels = [] for c in cervixes: cervixes_data.append(manager.buildFeatures(c['code_name'], configs)) cervixes_labels.append(c['type']) print('Feature length: ' + str(cervixes_data[0].shape)) FOLD_SIZE = 5 BATCH_SIZE = len(cervixes_data) // FOLD_SIZE WIN_PROB = 0.66 LOS_PROB = (1 - WIN_PROB) / 2 print('Start %d-fold cross-validation' % FOLD_SIZE)
from descriptor_manager import DescriptorManager import utilities as utils if (__name__ == '__main__'): wdir = '../../dataset/train-x512-processed-stage2' train_idx = os.path.join(wdir, 'idx_all.txt-shuf.txt') valid_idx = os.path.join(wdir, 'idx_test.txt') # valid_idx = 'idx-mask.txt-val.txt-processed.txt' print('Reading data...') train = utils.readCervixes(train_idx, "cervix", with_imgs=False) valid = utils.readCervixes(valid_idx, "test", with_imgs=False) # valid = utils.readCervixes('/home/ar/bitbucket.org/kaggle_intel_image_preprocessing-Dataset/train-x512-processed', valid_idx, "CNN") manager = DescriptorManager(os.path.join(wdir, 'descriptors')) configs = [ ['RGB-hist', {'bins': 32, 'mask': True}], ['RGB-hist', {'bins': 128, 'mask': True}], ['RGB-hist', {'bins': 256, 'mask': True}], ['LBP', {'radius': 12, 'numPoints': 36, 'mask': True}], ['LBP', {'radius': 16, 'numPoints': 48, 'mask': True}], ['LBP', {'radius': 32, 'numPoints': 160, 'mask': True}], # ['RGB-hist', {'bins': 32, 'mask': True}, "channel-mask"], # ['RGB-hist', {'bins': 128, 'mask': True}, "channel-mask"], # ['RGB-hist', {'bins': 256, 'mask': True}, "channel-mask"], # ['LBP', {'radius': 12, 'numPoints': 36, 'mask': True}, "channel-mask"], # ['LBP', {'radius': 16, 'numPoints': 48, 'mask': True}, "channel-mask"], # ['LBP', {'radius': 32, 'numPoints': 160, 'mask': True}, "channel-mask"], # ['KMeans', {'kmeans_model': True, 'mask': True}, "cervix-20000-512"],
class TimestampType(VariableType): def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCIDateTime) self.initialize_proc = self.initialize self.finalize_proc = self.finalize self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = None self.is_null_proc = None self.set_value_proc = self.set_value self.get_value_proc = self.get_value self.get_buffer_size_proc = None self.python_type = TIMESTAMP self.oracle_type = oci.SQLT_TIMESTAMP self.charset_form = oci.SQLCS_IMPLICIT self.size = ctypes.sizeof(self.oci_type) self.is_character_data = False self.is_variable_length = False self.can_be_copied = True self.can_be_in_array = True self.descriptor_manager = DescriptorManager() def initialize(self, var, cursor): self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_TIMESTAMP, "TimestampVar_Initialize()") def finalize(self, var): self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_TIMESTAMP) def set_value(self, var, pos, value): # make sure a timestamp is being bound if not isinstance(value, datetime): raise TypeError("expecting timestamp data") typed_data = self.get_typed_data(var) # store a copy of the value status = oci.OCIDateTimeConstruct(var.environment.handle, var.environment.error_handle, typed_data[pos], value.year, value.month, value.day, value.hour, value.minute, value.second, value.microsecond * 1000, None, 0) var.environment.check_for_error( status, "TimestampVar_SetValue(): create structure") c_invalid = oci.uword() status = oci.OCIDateTimeCheck(var.environment.handle, var.environment.error_handle, typed_data[pos], byref(c_invalid)) invalid = c_invalid.value var.environment.check_for_error(status, "TimestampVar_SetValue()") if invalid: raise DataError("invalid date") def get_value(self, var, pos): typed_data = self.get_typed_data(var) return oracle_timestamp_to_python_date(var.environment, typed_data[pos])
class BaseLobVariableType(VariableType): def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCILobLocator) self.initialize_proc = self.initialize self.finalize_proc = self.finalize self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = self.pre_fetch self.is_null_proc = None self.set_value_proc = self.set_value self.get_value_proc = self.get_value self.get_buffer_size_proc = None #self.python_type = None #self.oracle_type = None #self.charset_form = None self.size = ctypes.sizeof(self.oci_type) #self.is_character_data = None self.is_variable_length = False self.can_be_copied = False self.can_be_in_array = False self.descriptor_manager = DescriptorManager() def initialize(self, var, cursor): var.connection = cursor.connection self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_LOB, "LobVar_Initialize()") def finalize(self, var): self.pre_fetch(var) self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_LOB) def pre_fetch(self, var): """Free temporary LOBs prior to fetch.""" typed_data = self.get_typed_data(var) for i in xrange(var.allocelems): if typed_data[i]: c_is_temporary = ctypes.c_int() status = oci.OCILobIsTemporary(var.environment.handle, var.environment.error_handle, typed_data[i], byref(c_is_temporary)) var.environment.check_for_error( status, "LobVar_PreFetch(): is temporary LOB?") is_temporary = c_is_temporary.value # connection can be closed already, therefore we need to check for a non-null handle if is_temporary and var.connection.handle: status = oci.OCILobFreeTemporary( var.connection.handle, var.environment.error_handle, typed_data[i]) var.environment.check_for_error( status, "LobVar_PreFetch(): free temporary LOB") def set_value(self, var, pos, value): # make sure have temporary LOBs set up typed_data = self.get_typed_data(var) c_is_temporary = ctypes.c_int() status = oci.OCILobIsTemporary(var.environment.handle, var.environment.error_handle, typed_data[pos], byref(c_is_temporary)) var.environment.check_for_error(status, "LobVar_SetValue(): is temporary?") is_temporary = c_is_temporary.value if not is_temporary: if var.type.oracle_type == oci.SQLT_BLOB: lob_type = oci.OCI_TEMP_BLOB else: lob_type = oci.OCI_TEMP_CLOB status = oci.OCILobCreateTemporary(var.connection.handle, var.environment.error_handle, typed_data[pos], oci.OCI_DEFAULT, var.type.charset_form, lob_type, 0, oci.OCI_DURATION_SESSION) var.environment.check_for_error( status, "LobVar_SetValue(): create temporary") # trim the current value status = oci.OCILobTrim(var.connection.handle, var.environment.error_handle, typed_data[pos], 0) var.environment.check_for_error(status, "LobVar_SetValue(): trim") # set the current value self._write(var, pos, value, 1) def _write(self, var, pos, data_obj, offset): """Write data to the LOB variable.""" # verify the data type var_type = var.type if var_type == vt_BFILE: raise TypeError("BFILEs are read only") if var_type == vt_BLOB: buffer = cxBuffer.new_from_object(data_obj, var.environment.encoding) amount = buffer.size elif not python3_or_better() and var_type == vt_NCLOB: buffer = cxBuffer.new_from_object(data_obj, var.environment.nencoding) amount = buffer.size else: buffer = cxBuffer.new_from_object(data_obj, var.environment.encoding) if var.environment.fixedWidth and var.environment.maxBytesPerCharacter > 1: amount = buffer.size / var.environment.maxBytesPerCharacter else: amount = buffer.size # nothing to do if no data to write if amount == 0: return amount typed_data = self.get_typed_data(var) c_amount = oci.ub4(amount) callback_lob_write_type = oci.OCILobWrite.argtypes[9] null_oci_callback = callback_lob_write_type() status = oci.OCILobWrite(var.connection.handle, var.environment.error_handle, typed_data[pos], byref(c_amount), offset, buffer.ptr, buffer.size, oci.OCI_ONE_PIECE, None, null_oci_callback, 0, var.type.charset_form) var.environment.check_for_error(status, "LobVar_Write()") amount = c_amount.value # not sure if the write can change it, i dont think so, but just for correctness return amount def get_value(self, var, pos): return LOB(var, pos)
class TimestampType(VariableType): def __init__(self): VariableType.__init__(self) self.oci_type = oci.POINTER(oci.OCIDateTime) self.initialize_proc = self.initialize self.finalize_proc = self.finalize self.pre_define_proc = None self.post_define_proc = None self.pre_fetch_proc = None self.is_null_proc = None self.set_value_proc = self.set_value self.get_value_proc = self.get_value self.get_buffer_size_proc = None self.python_type = TIMESTAMP self.oracle_type = oci.SQLT_TIMESTAMP self.charset_form = oci.SQLCS_IMPLICIT self.size = ctypes.sizeof(self.oci_type) self.is_character_data = False self.is_variable_length = False self.can_be_copied = True self.can_be_in_array = True self.descriptor_manager = DescriptorManager() def initialize(self, var, cursor): self.descriptor_manager.initialize(self, var, cursor, oci.OCI_DTYPE_TIMESTAMP, "TimestampVar_Initialize()") def finalize(self, var): self.descriptor_manager.finalize(self, var, oci.OCI_DTYPE_TIMESTAMP) def set_value(self, var, pos, value): # make sure a timestamp is being bound if not isinstance(value, datetime): raise TypeError("expecting timestamp data") typed_data = self.get_typed_data(var) # store a copy of the value status = oci.OCIDateTimeConstruct(var.environment.handle, var.environment.error_handle, typed_data[pos], value.year, value.month, value.day, value.hour, value.minute, value.second, value.microsecond * 1000, None, 0); var.environment.check_for_error(status, "TimestampVar_SetValue(): create structure") c_invalid = oci.uword() status = oci.OCIDateTimeCheck(var.environment.handle, var.environment.error_handle, typed_data[pos], byref(c_invalid)) invalid = c_invalid.value var.environment.check_for_error(status, "TimestampVar_SetValue()") if invalid: raise DataError("invalid date") def get_value(self, var, pos): typed_data = self.get_typed_data(var) return oracle_timestamp_to_python_date(var.environment, typed_data[pos])