def clear(self): from utils import ObservableDict # only clear known variables for k in valid_vars & set(self.__dict__.keys()): del self.__dict__[k] self._groups = {} self._filters = ObservableDict() self._initialized = True
def LoadIncellFiles(self, properties_filename, sqlite_filename, incell_filenames): if os.path.exists(properties_filename): self.LoadFile(properties_filename) else: self._filename = properties_filename self._textfile = "" self._filters = ObservableDict() for incell_filename in incell_filenames: incell.parse_incell(sqlite_filename, incell_filename, self) self.Validate() self._initialized = True if not os.path.exists(properties_filename): self.save_file(properties_filename)
def Validate(self): '''Checks the validity of each field and their values. ''' # update old fields self.backwards_compatiblize() # check that all required fields are defined for name in required_vars: assert self.field_defined(name), 'PROPERTIES ERROR (%s): Field is missing or empty.'%(name) assert self.db_type.lower() in ['mysql', 'sqlite'], 'PROPERTIES ERROR (db_type): Value must be either "mysql" or "sqlite".' # BELOW: Check sometimes-optional fields, and print warnings etc if self.db_type.lower()=='sqlite': for field in ['db_port', 'db_host', 'db_name', 'db_user', 'db_passwd',]: if self.field_defined(field): logging.warn('PROPERTIES WARNING (%s): Field not required with db_type=sqlite.'%(field)) assert any([self.field_defined(field) for field in ['image_csv_file','object_csv_file','db_sql_file','db_sqlite_file']]), \ 'PROPERTIES ERROR: When using db_type=sqlite, you must also supply the fields "image_csv_file" and "object_csv_file" OR "db_sql_file" OR "db_sqlite_file". See the README.' if self.field_defined('db_sqlite_file'): if not os.path.isabs(self.db_sqlite_file): # Make relative paths relative to the props file location # TODO: This sholdn't be permanent self.db_sqlite_file = os.path.join(os.path.dirname(self._filename), self.db_sqlite_file) try: f = open(self.db_sqlite_file, 'r') f.close() except: raise Exception, 'PROPERTIES ERROR (%s): SQLite database could not be found at "%s".'%('db_sqlite_file', self.db_sqlite_file) if self.field_defined('db_sql_file'): if not os.path.isabs(self.db_sql_file): # Make relative paths relative to the props file location # TODO: This sholdn't be permanent self.db_sql_file = os.path.join(os.path.dirname(self._filename), self.db_sql_file) try: f = open(self.db_sql_file, 'r') f.close() except: raise Exception, 'PROPERTIES ERROR (%s): File "%s" could not be found.'%('db_sql_file', self.db_sql_file) for field in ['image_csv_file','object_csv_file']: assert not self.field_defined(field), 'PROPERTIES ERROR (%s, db_sql_file): Both of these fields cannot be used at the same time.'%(field) else: for field in ['image_csv_file','object_csv_file']: if self.field_defined(field): if not os.path.isabs(self.__dict__[field]): # Make relative paths relative to the props file location # TODO: This sholdn't be permanent self.__dict__[field] = os.path.join(os.path.dirname(self._filename), self.__dict__[field]) try: f = open(self.__dict__[field], 'r') f.close() except: raise Exception, 'PROPERTIES ERROR (%s): File "%s" could not be found.'%(field, self.__dict__[field]) if self.db_type.lower()=='mysql': for field in ['db_host', 'db_name', 'db_user',]: assert self.field_defined(field), 'PROPERTIES ERROR (%s): Field is required with db_type=mysql.'%(field) if not self.field_defined('db_port'): self.db_port = '3306' logging.info('PROPERTIES: Using default db_port=3306 for MySQL.') for field in ['image_csv_file','object_csv_file']: if self.field_defined(field): logging.warn('PROPERTIES WARNING (%s): Field not required with db_type=mysql.'%(field)) if self.field_defined('area_scoring_column'): logging.info('PROPERTIES: Area scoring will be used.') if not self.field_defined('image_channel_colors'): logging.warn('PROPERTIES WARNING (image_channel_colors): No value(s) specified. CPA will use a generic channel-color mapping.') self.image_channel_colors = ['red', 'green', 'blue']+['none' for x in range(97)] if not self.field_defined('channels_per_image'): logging.warn('PROPERTIES WARNING (channels_per_image): No value(s) specified. CPA will assume 1 channel per image.') self.channels_per_image = ['1' for i in range(len(self.image_file_cols))] if not self.field_defined('image_names'): logging.warn('PROPERTIES WARNING (image_names): No value(s) specified. CPA will use generic channel names.') self.image_names = ['channel-%d'%(i+1) for i in range(len(self.image_file_cols))] if len(self.image_channel_colors) < sum(map(int, self.channels_per_image)): self.image_channel_colors = ['red', 'green', 'blue'] self.image_channel_colors += ['none' for x in range(min(sum(map(int, self.channels_per_image)) - 3, 0))] logging.warn('PROPERTIES WARNING (image_channel_colors): You did not ' 'specify enough colors for all the channels in your images. ' 'One color should be listed for each file column listed in ' 'image_file_cols unless your images contain multiple ' 'channels, in which case you need one for the sum of all ' 'the channels specified by "channels_per_image". CPA, will ' 'use channel colors %s for this run.'%(self.image_channel_colors,)) assert len(self.image_file_cols) == len(self.image_path_cols), \ 'PROPERTIES ERROR: image_file_cols and image_path_cols must have an equal number of values.' assert len(self.image_file_cols) == len(self.channels_per_image), \ 'PROPERTIES ERROR: channels_per_image must have the same number of values as image_file_cols and image_path_cols.' assert len(self.image_file_cols) == len(self.image_names), \ 'PROPERTIES ERROR: image_names must have the same number of values as image_file_cols and image_path_cols.' if self.field_defined('image_channel_blend_modes'): for mode in self.image_channel_blend_modes: assert mode in ['add', 'subtract', 'solid'], 'PROPERTIES ERROR (image_channel_blend_modes): Blend modes must list of modes (1 for each image channel). Valid modes are add, subtract and solid.' if not self.field_defined('classifier_ignore_columns'): logging.warn('PROPERTIES WARNING (classifier_ignore_columns): No value(s) specified. Classifier will use ALL NUMERIC per_object columns when training.') if not self.field_defined('image_buffer_size'): logging.info('PROPERTIES: Using default image_buffer_size=1') self.image_buffer_size = '1' if not self.field_defined('tile_buffer_size'): logging.info('PROPERTIES: Using default tile_buffer_size=1') self.tile_buffer_size = '1' if not self.field_defined('object_name'): logging.warn('PROPERTIES WARNING (object_name): No object name specified, will use default: "object_name=cell,cells"') self.object_name = ['cell', 'cells'] else: # if it is defined make sure they do it correctly assert len(self.object_name)==2, 'PROPERTIES ERROR (object_name): Found %d names instead of 2! This field should contain the singular and plural name of the objects you are classifying. (Example: object_name=cell,cells)'%(len(self.object_name)) if self.field_defined('training_set'): if not os.path.isabs(self.training_set): # Make relative paths relative to the props file location # TODO: This sholdn't be permanent self.training_set = os.path.join(os.path.dirname(self._filename), self.training_set) try: f = open(self.training_set) f.close() except: logging.warn('PROPERTIES WARNING (training_set): Training set at "%s" could not be found.'%(self.training_set)) logging.info('PROPERTIES: Training set found at "%s"'%(self.training_set)) if self.field_defined('class_table'): assert self.class_table != self.image_table, 'PROPERTIES ERROR (class_table): class_table cannot be the same as image_table!' assert self.class_table != self.object_table, 'PROPERTIES ERROR (class_table): class_table cannot be the same as object_table!' logging.info('PROPERTIES: Per-Object classes will be written to table "%s"'%(self.class_table)) if not self.field_defined('plate_id'): logging.warn('PROPERTIES WARNING (plate_id): Field is required for plate map viewer.') if not self.field_defined('well_id'): logging.warn('PROPERTIES WARNING (well_id): Field is required for plate map viewer.') # # plate_shape # This field can be set directly, otherwise it is set indirectly by the plate_type field # if self.field_defined('plate_shape'): if len(self.__dict__['plate_shape']) != 2: raise Exception('PROPERTIES ERROR: invalid value (%s) for plate_shape. Expected: rows, cols' %(self.__dict__['plate_shape'])) try: self.__dict__['plate_shape'][0] = int(self.__dict__['plate_shape'][0]) self.__dict__['plate_shape'][1] = int(self.__dict__['plate_shape'][1]) except: raise Exception('PROPERTIES ERROR: invalid value (%s) for plate_shape. Expected: rows, cols' %(self.__dict__['plate_shape'])) # # plate_type # This field is used to set the plate_shape field indirectly # if not self.field_defined('plate_type'): logging.warn('PROPERTIES WARNING (plate_type): Field is required for plate viewer') else: if self.field_defined('plate_shape'): raise Exception('PROPERTIES ERROR: You have defined plate_type and plate_shape in your properties file. Please use one or the other.') if self.__dict__['plate_type'] not in supported_plate_types.keys(): raise Exception('PROPERTIES ERROR: invalid value (%s) for plate_type. Supported plate type are: %s.' %(self.__dict__['plate_type'], ', '.join(supported_values))) self.plate_shape = supported_plate_types[self.__dict__['plate_type']] if self.field_defined('check_tables') and self.check_tables.lower() in ['false', 'no', 'off', 'f', 'n']: self.check_tables = 'no' elif not self.field_defined('check_tables') or self.check_tables.lower() in ['true', 'yes', 'on', 't', 'y']: self.check_tables = 'yes' else: logging.warn('PROPERTIES WARNING (check_tables): Field value "%s" is invalid. Replacing with "yes".'%(self.check_tables)) self.check_tables = 'yes' if self.use_larger_image_scale in [True, False]: pass elif not self.field_defined('use_larger_image_scale') or self.use_larger_image_scale.lower() in ['false', 'no', 'off', 'f', 'n']: self.use_larger_image_scale = False elif self.field_defined('use_larger_image_scale') and self.use_larger_image_scale.lower() in ['true', 'yes', 'on', 't', 'y']: self.use_larger_image_scale = True else: logging.warn('PROPERTIES WARNING (use_larger_image_scale): Field value "%s" is invalid. Replacing with "false".'%(self.use_larger_image_scale)) self.use_larger_image_scale = False if self.rescale_object_coords in [True, False]: pass elif not self.field_defined('rescale_object_coords') or self.rescale_object_coords.lower() in ['false', 'no', 'off', 'f', 'n']: self.rescale_object_coords = False elif self.field_defined('rescale_object_coords') and self.rescale_object_coords.lower() in ['true', 'yes', 'on', 't', 'y']: self.rescale_object_coords = True else: logging.warn('PROPERTIES WARNING (rescale_object_coords): Field value "%s" is invalid. Replacing with "false".'%(self.rescale_object_coords)) self.rescale_object_coords = False if not self.field_defined('well_format'): self.well_format = 'A01' logging.warn('PROPERTIES WARNING (well_format): Field was not defined, using default format of "A01".') if not self.field_defined('link_tables_table'): self.link_tables_table = '_link_tables_%s_%s_'%(self.image_table, (self.object_table or '')) if len(self.link_tables_table) >= 64: from hashlib import md5 self.link_tables_table = '_link_tables_%s'%(md5(self.image_table+(self.object_table or '')).hexdigest()) if not self.field_defined('link_columns_table'): self.link_columns_table = '_link_columns_%s_%s_'%(self.image_table, (self.object_table or '')) if len(self.link_columns_table) >= 64: from hashlib import md5 self.link_columns_table = '_link_columns_%s'%(md5(self.image_table+(self.object_table or '')).hexdigest()) if not self.field_defined('gates'): from utils import ObservableDict self.gates = ObservableDict()
def load_file(self, filename): ''' Loads variables in from a properties file. ''' from sqltools import Gate, Filter, OldFilter self.clear() self._filename = filename self._groups = {} self._filters = ObservableDict() self.gates = ObservableDict() f = open(filename, 'U') lines = f.read() self._textfile = lines # store raw file lines = lines.split('\n') for idx in xrange(len(lines)): line = lines[idx] # skip commented and empty lines if not line.strip().startswith('#') and line.strip() != '': try: (name, val) = line.split('=', 1) name = name.strip() val = val.strip() except ValueError: raise Exception('PROPERTIES ERROR: Could not parse line #%d\n' '\t"%s"\n' 'Did you accidentally load your training set instead of your properties file?'%(idx + 1, line)) if name in string_vars: self.__dict__[name] = val or None elif name in list_vars: self.__dict__[name] = self.parse_list_value(val) or None elif name.startswith('group_SQL_'): group_name = name[10:] if group_name == '': raise Exception, ('PROPERTIES ERROR (%s): "group_SQL_" should be followed by a group name.\n' 'Example: "group_SQL_MyGroup = <QUERY>" would define a group named "MyGroup" defined by\n' 'a MySQL query "<QUERY>". See the README.'%(name)) if group_name in self._groups.keys(): raise Exception, 'Group "%s" is defined twice in properties file.'%(group_name) if group_name in self._filters.keys(): raise Exception, 'Name "%s" is already taken for a filter.'%(group_name) if not val: logging.warn('PROPERTIES WARNING (%s): Undefined group'%(name)) continue self._groups[group_name] = val elif name.startswith('filter_SQL_'): # Load old-style SQL filters: filter_name = name[11:] if filter_name == '': raise Exception, ('PROPERTIES ERROR (%s): "filter_SQL_" should be followed by a filter name.\n' 'Example: "filter_SQL_MyFilter = <QUERY>" would define a filter named "MyFilter" defined by\n' 'a MySQL query "<QUERY>". See the README.'%(name)) if filter_name in self._filters.keys(): raise Exception, 'Filter "%s" is defined twice in properties file.'%(filter_name) if filter_name in self._groups.keys(): raise Exception, 'Name "%s" is already taken for a group.'%(filter_name) if re.search('\W', filter_name): raise Exception, 'PROPERTIES ERROR (%s): Filter names may only contain alphanumeric characters and "_".'%(filter_name) if not val: logging.warn('PROPERTIES WARNING (%s): Undefined filter'%(name)) continue self._filters[filter_name] = OldFilter(val) elif name == 'groups': logging.warn('PROPERTIES WARNING (%s): This field is no longer necessary in the properties file.\n' 'Only the group_SQL_XXX and filter_SQL_XXX fields are needed when defining groups and filters.'%(name)) elif name == 'filters': # Load new-style filters if val.strip() == '': logging.warn('PROPERTIES WARNING (filters): Field should not be left blank') continue d = eval(val) if type(d) != dict: raise Exception, 'PROPERTIES ERROR (filters): Error parsing filters. Check the "filters" field in your properties file.' for k, v in d.items(): self._filters[k] = Filter.decode(v) del d elif name == 'gates': d = eval(val) if type(d) != dict: raise Exception, 'PROPERTIES ERROR (gates): Error parsing gates. Check the "gates" field in your properties file.' for k, v in d.items(): self.gates[k] = Gate.decode(v) del d else: logging.warn('PROPERTIES WARNING: Unrecognized field "%s" in properties file'%(name)) f.close() self.Validate() self._initialized = True
def load_file(self, filename): ''' Loads variables in from a properties file. ''' from sqltools import Gate, Filter, OldFilter self.clear() self._filename = filename self._groups = {} self._filters = ObservableDict() self.gates = ObservableDict() f = open(filename, 'U') lines = f.read() self._textfile = lines # store raw file lines = lines.split('\n') for idx in xrange(len(lines)): line = lines[idx] # skip commented and empty lines if not line.strip().startswith('#') and line.strip() != '': try: (name, val) = line.split('=', 1) name = name.strip() val = val.strip() except ValueError: raise Exception( 'PROPERTIES ERROR: Could not parse line #%d\n' '\t"%s"\n' 'Did you accidentally load your training set instead of your properties file?' % (idx + 1, line)) if name in string_vars: self.__dict__[name] = val or None elif name in list_vars: self.__dict__[name] = self.parse_list_value(val) or None elif name.startswith('group_SQL_'): group_name = name[10:] if group_name == '': raise Exception, ( 'PROPERTIES ERROR (%s): "group_SQL_" should be followed by a group name.\n' 'Example: "group_SQL_MyGroup = <QUERY>" would define a group named "MyGroup" defined by\n' 'a MySQL query "<QUERY>". See the README.' % (name)) if group_name in self._groups.keys(): raise Exception, 'Group "%s" is defined twice in properties file.' % ( group_name) if group_name in self._filters.keys(): raise Exception, 'Name "%s" is already taken for a filter.' % ( group_name) if not val: logging.warn( 'PROPERTIES WARNING (%s): Undefined group' % (name)) continue self._groups[group_name] = val elif name.startswith('filter_SQL_'): # Load old-style SQL filters: filter_name = name[11:] if filter_name == '': raise Exception, ( 'PROPERTIES ERROR (%s): "filter_SQL_" should be followed by a filter name.\n' 'Example: "filter_SQL_MyFilter = <QUERY>" would define a filter named "MyFilter" defined by\n' 'a MySQL query "<QUERY>". See the README.' % (name)) if filter_name in self._filters.keys(): raise Exception, 'Filter "%s" is defined twice in properties file.' % ( filter_name) if filter_name in self._groups.keys(): raise Exception, 'Name "%s" is already taken for a group.' % ( filter_name) if re.search('\W', filter_name): raise Exception, 'PROPERTIES ERROR (%s): Filter names may only contain alphanumeric characters and "_".' % ( filter_name) if not val: logging.warn( 'PROPERTIES WARNING (%s): Undefined filter' % (name)) continue self._filters[filter_name] = OldFilter(val) elif name == 'groups': logging.warn( 'PROPERTIES WARNING (%s): This field is no longer necessary in the properties file.\n' 'Only the group_SQL_XXX and filter_SQL_XXX fields are needed when defining groups and filters.' % (name)) elif name == 'filters': # Load new-style filters if val.strip() == '': logging.warn( 'PROPERTIES WARNING (filters): Field should not be left blank' ) continue d = eval(val) if type(d) != dict: raise Exception, 'PROPERTIES ERROR (filters): Error parsing filters. Check the "filters" field in your properties file.' for k, v in d.items(): self._filters[k] = Filter.decode(v) del d elif name == 'gates': d = eval(val) if type(d) != dict: raise Exception, 'PROPERTIES ERROR (gates): Error parsing gates. Check the "gates" field in your properties file.' for k, v in d.items(): self.gates[k] = Gate.decode(v) del d else: logging.warn( 'PROPERTIES WARNING: Unrecognized field "%s" in properties file' % (name)) f.close() #if classification_type is defined if self.field_defined('classification_type' ) and self.classification_type.lower() in [ 'image' ]: self.classification_type = 'image' #2 cases: # object_table/object_id/cell_x_loc/cell_y_loc exist and point to a table/view of object tables # object_table/object_id/cell_x_loc/cell_y_loc don't exist self.object_table = self.image_table + '_ObjectTable' self.object_id = 'ObjectNumber' self.cell_x_loc = 'Image_x_loc' self.cell_y_loc = 'Image_y_loc' self.object_name = ['image', 'images'] # image_width and image_height refer to the full image width and height while # image_tile_size refers to the size of tiles for classification # NB: if image_width and image_height are defined, they override image_tile_size if self.field_defined('image_width') and self.field_defined( 'image_height'): self.image_width, self.image_height = int( self.image_width), int(self.image_height) self.image_tile_size = min( [self.image_width, self.image_height]) self.image_tile_size = int(self.image_tile_size) else: self.classification_type = "object" # For image gallery if self.field_defined('image_size'): self.image_size = int(self.image_size) else: self.image_size = self.image_tile_size self.Validate() #if check_tables is defined if self.field_defined( 'check_tables') and self.check_tables.lower() in ['yes']: self.object_table = self.object_table + '_Checked' self._initialized = True