def _create_es_index(project_id, data_params, module_params): ''' Create an Elasticsearch index for the selected file GET: - project_id: Link project_id POST: - data_params: { project_type: (optional) defaults to link module_name: file_name: } - module_params: { columns_to_index: for_linking: create index to use as referential (instead of storage) force: force recreation of index even if existant } ''' if module_params is None: module_params = {} print(module_params) columns_to_index = module_params.get('columns_to_index') force = module_params.get('force', False) for_linking = module_params.get('for_linking', True) if (not for_linking) and (columns_to_index is not None): raise ValueError( 'columns_to_index and for_linking cannot be not None and False') if (data_params is not None) and ('project_type' in data_params): project_type = data_params['project_type'] project_type = 'link' if data_params is not None: module_name = data_params['module_name'] file_name = data_params['file_name'] project_type = data_params.get('project_type', 'link') # TODO: dirty fix for linking and normalization if for_linking: if project_type == 'link': proj_link = ESLinker(project_id) proj = ESNormalizer(proj_link.ref.project_id) if data_params is None: module_name = proj_link.metadata['files']['ref']['module_name'] file_name = proj_link.metadata['files']['ref']['file_name'] elif project_type == 'normalize': proj = ESNormalizer(project_id) # Generate default columns_to_index if columns_to_index is None: columns_to_index = proj.gen_default_columns_to_index(for_linking) else: proj = ESLinker(project_id) if data_params is None: module_name, file_name = proj.get_last_written() if columns_to_index is None: columns_to_index = { col: {} for col in proj._get_header(module_name, file_name) } file_path = proj.path_to(module_name, file_name) proj.create_index(file_path, columns_to_index, force) time.sleep(5) # TODO: why is this necessary? return
class Linker(ESAbstractDataProject): MODULES = LINK_MODULES MODULE_ORDER = LINK_MODULE_ORDER MODULE_ORDER_log = LINK_MODULE_ORDER_log def __init__(self, project_id=None, create_new=False, display_name=None, description=None, public=False): super().__init__(project_id, create_new, display_name=display_name, description=description, public=public) # Add source and ref if the were selected if (self.metadata['files']['source'] is not None) \ and (self.metadata['files']['ref'] is not None): self.load_project_to_merge('source') self.load_project_to_merge('ref') def __repr__(self): string = '{0}({1})'.format(self.__class__.__name__, self.project_id) string += ' / source: ' if self.source is not None: string += self.source.__repr__() else: string += 'None' string += ' / ref: ' if self.ref is not None: string += self.ref.__repr__() return string def __str__(self): string = '{0}; project_id:{1}'.format(self.__class__.__name__, self.project_id) if self.source is not None: string += '\n\n***SOURCE***\n{0}'.format(self.source.__str__()) if self.ref is not None: string += '\n\n***REF***\n{0}'.format(self.ref.__str__()) return string @staticmethod def output_file_name(source_file_name): '''Name of the file to output''' return source_file_name def load_project_to_merge(self, file_role): '''Uses the "current" field in metadata to load source or ref''' self._check_file_role(file_role) # TODO: Add safeguard somewhere # Add source if file_role == 'source': try: self.source = ESNormalizer( self.metadata['files']['source']['project_id']) except: self.source = None if file_role == 'ref': try: self.ref = ESNormalizer( self.metadata['files']['ref']['project_id']) except: self.ref = None #raise Exception('Normalizer project with id {0} could not be found'.format(project_id)) @staticmethod def _check_file_role(file_role): if file_role not in ['ref', 'source']: raise Exception('file_role should be either "source" or "ref"') def _check_select(self): '''Check that a source and referential were selected''' for file_role in ['source', 'ref']: if self.metadata['files'][file_role] is None: raise Exception( '{0} is not defined for this linking project'.format( file_role)) def _create_metadata(self, description=None, display_name=None, public=False): metadata = super()._create_metadata(description=description, display_name=display_name, public=public) metadata['files'] = { 'source': None, 'ref': None } # {'source': {public: False, project_id: "ABC123", file_name: "source.csv.csv"}, 'ref': None} metadata['project_type'] = 'link' return metadata def add_col_matches(self, column_matches): ''' Adds a configuration file with the column matches between source and referential. INPUT: - column_matches: json file as dict ''' # TODO: add checks on file if (self.source is None) or (self.ref is None): raise RuntimeError( 'source or referential were not loaded (add_selected_project) and/or (load_project_to_merge)' ) # Add matches self.upload_config_data(column_matches, 'es_linker', 'column_matches.json') # Select these columns for normalization in source and ref # TODO: this will cover add_certain_col_matches # Add to log for file_name in self.metadata['log']: self.metadata['log'][file_name]['add_selected_columns'][ 'completed'] = True self._write_metadata() def add_es_learned_settings(self, learned_settings): '''Adds the learned es configuration''' print('trying to upload', learned_settings) self.upload_config_data(learned_settings, 'es_linker', 'learned_settings.json') for file_name in self.metadata['log']: self.metadata['log'][file_name]['upload_es_train'][ 'completed'] = True self._write_metadata() def read_col_matches(self, add_created=True): ''' Read the column_matches config file and interprets the columns looking for processed (normalized) columns ''' config = self.read_config_data('es_linker', 'column_matches.json') if not config: config = [] return config def add_col_certain_matches(self, column_matches): '''column_matches is a json file as list of dict of list''' # TODO: add checks on file self.upload_config_data(column_matches, 'es_linker', 'column_certain_matches.json') def read_col_certain_matches(self): config = self.read_config_data('es_linker', 'column_certain_matches.json') if not config: config = [] return config def add_cols_to_return(self, file_role, columns): ''' columns is a list of columns in the referential that we want to return during download ''' # Check that both projects are finished for file_role in ['source', 'ref']: file_name = self.metadata['files'][file_role]['file_name'] if not self.__dict__[file_role].metadata['complete'][file_name]: raise Exception('Cannot select columns: complete {0} project \ ({1}) before...'.format( file_role, self.__dict__[file_role].project_id)) # Write columns to return to config config_file_name = 'columns_to_return_{0}.json'.format(file_role) self.upload_config_data(columns, 'es_linker', config_file_name) def read_cols_to_return(self, file_role): config_file_name = 'columns_to_return_{0}.json'.format(file_role) config = self.read_config_data('es_linker', config_file_name) if not config: config = [] return config def add_selected_project(self, file_role, public, project_id): ''' Select file to use as source or referential. INPUT: - file_role: "source" or "referential" - public: (bool) is the project available to all (or is it a user project) - project_id - file_name ''' self._check_file_role(file_role) # Check that file exists if public: raise DeprecationWarning else: proj = ESNormalizer(project_id) # if file_name not in proj.metadata['files']: # raise Exception('File {0} could not be found in project {1} \ # (public: {2})'.format(file_name, project_id, public)) # Check that normalization project has only one file (and possibly a MINI__ version) if not len(proj.metadata['files']): raise Exception( 'The selected normalization project ({0}) has no upload file'. format(project_id)) if len(proj.metadata['files']) > 1: raise Exception('The selected normalization project ({0}) has more than one file.'\ + ' This method expects projects to have exactly 1 file as it'\ + ' uses the implicit get_last_written'.format(project_id)) # TODO: last written is a bad idea because if we modify normalization then BOOM ! # TODO: last_written btw concat_with_initi and init ? (module_name, file_name) = proj.get_last_written() # TODO: add warning for implicit use of not-MINI if proj.metadata['has_mini'] and (file_role == 'source'): file_name = file_name.replace('MINI__', '') if proj.metadata['has_mini'] and (file_role == 'ref'): file_name = file_name.replace('MINI__', '') # Check that self.metadata['files'][file_role] = { 'public': public, 'project_id': project_id, 'module_name': module_name, 'file_name': file_name, 'restricted': False } # Create log for source if file_role == 'source': self.metadata['log'][self.output_file_name( file_name)] = self._default_log() # Add project selection if (self.metadata['files']['source'] is not None) and (self.metadata['files']['ref'] is not None): for file_name in self.metadata['log']: self.metadata['log'][file_name]['INIT']['completed'] = True self._write_metadata() self.load_project_to_merge(file_role) def read_selected_files(self): ''' Returns self.metadata['files'] ''' return self.metadata['files'] def infer(self, module_name, params): '''Overwrite to allow restrict_reference''' if module_name == 'infer_restriction': params['NO_MEM_DATA'] = True return super().infer(module_name, params) def linker(self, module_name, data_params, module_params): '''Wrapper aro''' if module_name == 'es_linker': return self.es_linker(module_params) elif module_name == 'dedupe_linker': raise DeprecationWarning def es_linker(self, module_params): module_params['index_name'] = ESNormalizer( self.ref.project_id).index_name self.source.load_data(*self.source.get_last_written()) self.mem_data = self.source.mem_data self.mem_data_info = self.source.mem_data_info # Change file_name to output file_name self.mem_data_info['file_name'] = self.output_file_name( self.mem_data_info['file_name']) # File being modified log, run_info = self.transform('es_linker', module_params) return log, run_info def write_labeller(self, module_name, labeller): '''Pickles the labeller object in project''' # TODO: Add isinstance(labeller, Labeller) pickle_path = self.path_to(module_name, 'labeller.pkl') labeller.to_pickle(pickle_path) def _read_labeller(self, module_name): '''Reads labeller stored in pickle''' pickle_path = self.path_to(module_name, 'labeller.pkl') labeller = ESLabeller.from_pickle(pickle_path, es) return labeller #========================================================================== # Module specific: ES Linker #========================================================================== def _gen_paths_es(self): self._check_select() # Get path to training file for ES linker training_path = self.path_to('es_linker', 'training.json') learned_settings_path = self.path_to('es_linker', 'learned_settings.json') # TODO: check that normalization projects are complete ? # Get path to source # TODO: fix this: use current file_name = self.metadata['files']['source']['file_name'] source_path = self.source.path_to_last_written(module_name=None, file_name=file_name) # Add paths paths = { 'source': source_path, 'train': training_path, 'learned_settings': learned_settings_path } return paths @staticmethod def _tuple_or_string(x): if isinstance(x, str): return x elif isinstance(x, list): if len(x) == 1: return x[0] else: return tuple(x) elif isinstance(x, tuple): if len(x) == 1: return x[0] else: return x else: raise ValueError('Value should be str, list or tuple') def _gen_es_labeller(self, columns_to_index=None, certain_column_matches=None): ''' Return a es_labeller object ''' self._check_select() #chunksize = 40000 col_matches_tmp = self.read_col_matches() col_matches = [] for match in col_matches_tmp: col_matches.append({ 'source': self._tuple_or_string(match['source']), 'ref': self._tuple_or_string(match['ref']) }) # TODO: lists to tuple in col_matches paths = self._gen_paths_es() source = pd.read_csv(paths['source'], sep=',', encoding='utf-8', dtype=str, nrows=3000) source = source.where(source.notnull(), '') ref_table_name = self.ref.project_id if columns_to_index is None: columns_to_index = self.ref.gen_default_columns_to_index() labeller = ESLabeller(es, source, ref_table_name, col_matches, columns_to_index, certain_column_matches) # TODO: Auto label certain pairs # TODO: Add pre-load for 3 first queries return labeller def labeller_to_json(self, labeller): file_path = self.path_to('es_linker', 'labeller.json') labeller.to_json(file_path) def labeller_from_json(self): file_path = self.path_to('es_linker', 'labeller.json') paths = self._gen_paths_es() source = pd.read_csv(paths['source'], sep=',', encoding='utf-8', dtype=str, nrows=3000) source = source.where(source.notnull(), '') ref_table_name = self.ref.project_id labeller = ESLabeller.from_json(file_path, es, source, ref_table_name) return labeller def analyze_results(self, params={}): # Check that memory is loaded (if necessary) self._check_mem_data() module_name = 'link_results_analyzer' # Initiate log log = self._init_active_log(module_name, 'infer') agg_results = defaultdict(int) for data in self.mem_data: infered_params = link_results_analyzer(data, params) agg_results['num_match'] += infered_params['num_match'] agg_results['num_match_thresh'] += infered_params[ 'num_match_thresh'] # Write result of inference module_to_write_to = self.MODULES['infer'][module_name]['write_to'] self.upload_config_data(agg_results, module_to_write_to, 'infered_config.json') # Update log buffer self._end_active_log(log, error=False) return infered_params