def __init__(self, options, columns): super(FilesystemFdw, self).__init__(options, columns) root_dir = options.get('root_dir') pattern = options.get('pattern') self.content_column = options.get('content_column', None) self.filename_column = options.get('filename_column', None) self.structured_directory = StructuredDirectory(root_dir, pattern) if self.filename_column: if self.filename_column not in columns: log_to_postgres("The filename column (%s) does not exist" "in the column list" % self.filename_column, ERROR, "You should try to create your table with an " "additional column: \n" "%s character varying" % self.filename_column) else: columns.remove(self.filename_column) if self.content_column: if self.content_column not in columns: log_to_postgres("The content column (%s) does not exist" "in the column list" % self.content_column, ERROR, "You should try to create your table with an " "additional column: \n" "%s bytea" % self.content_column) else: columns.remove(self.content_column) if len(self.structured_directory.properties) < len(columns): missing_columns = set(columns).difference( self.structured_directory.properties) log_to_postgres("Some columns are not mapped in the structured fs", WARNING, "Remove the following columns: %s " % missing_columns)
def __init__(self, options, columns): super(FilesystemFdw, self).__init__(options, columns) root_dir = options.get('root_dir') pattern = options.get('pattern') self.content_column = options.get('content_column', None) self.filename_column = options.get('filename_column', None) self.file_mode = int(options.get('file_mode', '700'), 8) self.structured_directory = StructuredDirectory( root_dir, pattern, file_mode=self.file_mode) self.folder_columns = [ key[0] for key in self.structured_directory._path_parts_properties if key ] # Keep a set of files that should not be seen inside the transaction, # because they have "logically" been deleted, but are not yet commited self.invisible_files = set() # Keep a dictionary of updated content. self.updated_content = dict() # Assume 100 files/folder per folder self.total_files = 100**len(pattern.split('/')) if self.filename_column: if self.filename_column not in columns: log_to_postgres( "The filename column (%s) does not exist" "in the column list" % self.filename_column, ERROR, "You should try to create your table with an " "additional column: \n" "%s character varying" % self.filename_column) else: columns.pop(self.filename_column) if self.content_column: if self.content_column not in columns: log_to_postgres( "The content column (%s) does not exist" "in the column list" % self.content_column, ERROR, "You should try to create your table with an " "additional column: \n" "%s bytea" % self.content_column) else: columns.pop(self.content_column) if len(self.structured_directory.properties) < len(columns): missing_columns = set(columns.keys()).difference( self.structured_directory.properties) log_to_postgres("Some columns are not mapped in the structured fs", level=WARNING, hint="Remove the following columns: %s " % missing_columns)