def setup_sql_load(self): """Setup SQL DB creation""" for table in self.tables: if table == "words": file_in = self.destination + "/WORK/all_words_ordered" indices = [("philo_name",), ("philo_id",), ("parent",), ("start_byte",), ("end_byte",)] depth = 7 elif table == "pages": file_in = self.destination + "/WORK/all_pages" indices = [("philo_id",)] depth = 9 elif table == "toms": file_in = self.destination + "/WORK/all_toms_sorted" indices = [("philo_type",), ("philo_id",), ("img",)] + self.metadata_fields depth = 7 elif table == "refs": file_in = self.destination + "/WORK/all_refs" indices = [("parent",), ("target",), ("type",)] depth = 9 elif table == "graphics": file_in = self.destination + "/WORK/all_graphics" indices = [("parent",), ("philo_id",)] depth = 9 elif table == "lines": file_in = self.destination + "/WORK/all_lines" indices = [("doc_id", "start_byte", "end_byte")] depth = 9 post_filter = make_sql_table(table, file_in, indices=indices, depth=depth) self.post_filters.insert(0, post_filter)
def setup_sql_load(self): for table in self.tables: if table == 'words': file_in = self.destination + '/WORK/all_words_ordered' indices = [("philo_name", ), ('philo_id', ), ('parent', ), ('start_byte', ), ('end_byte', )] depth = 7 elif table == 'pages': file_in = self.destination + '/WORK/all_pages' indices = [("philo_id", )] depth = 9 elif table == 'toms': file_in = self.destination + '/WORK/all_toms_sorted' indices = [('philo_type', ), ('philo_id', ), ('img', )] + self.metadata_fields depth = 7 elif table == "refs": file_in = self.destination + '/WORK/all_refs' indices = [("parent", ), ("target", ), ("type", )] depth = 9 elif table == "graphics": file_in = self.destination + '/WORK/all_graphics' indices = [("parent", ), ("philo_id", )] depth = 9 elif table == "lines": file_in = self.destination + '/WORK/all_lines' indices = [("doc_id", "start_byte", "end_byte")] depth = 9 post_filter = make_sql_table(table, file_in, indices=indices, depth=depth) self.post_filters.insert(0, post_filter)
def setup_sql_load(self): for table in self.tables: if table == 'words': file_in = self.destination + '/WORK/all_words_sorted.gz' indices = [("philo_name", "%s_ancestor" % self.default_object_level), ('philo_id', )] depth = 7 compressed = True elif table == 'pages': file_in = self.destination + '/WORK/all_pages' indices = [("philo_id", )] depth = 9 compressed = False elif table == 'toms': file_in = self.destination + '/WORK/all_toms_sorted' indices = [('philo_type', ), ('philo_id', )] + self.metadata_fields depth = 7 compressed = False post_filter = make_sql_table(table, file_in, gzip=compressed, indices=indices, depth=depth) self.post_filters.insert(0, post_filter)
def setup_sql_load(self): for table in self.tables: if table == 'words': file_in = self.destination + '/WORK/all_words_sorted.gz' indices = [("philo_name", "%s_ancestor" % self.default_object_level), ('philo_id',)] depth = 7 compressed = True elif table == 'pages': file_in = self.destination + '/WORK/all_pages' indices = [("philo_id",)] depth = 9 compressed = False elif table == 'toms': file_in = self.destination + '/WORK/all_toms_sorted' indices = [('philo_type',), ('philo_id',)] + self.metadata_fields depth = 7 compressed = False post_filter = make_sql_table(table, file_in, gzip=compressed, indices=indices, depth=depth) self.post_filters.insert(0, post_filter)
def setup_sql_load(self): for table in self.tables: if table == 'words': file_in = self.destination + '/WORK/all_words_ordered' indices = [("philo_name",), ('philo_id',), ('parent',), ('byte_start',), ('byte_end',)] depth = 7 compressed = False elif table == 'pages': file_in = self.destination + '/WORK/all_pages' indices = [("philo_id",)] depth = 9 compressed = False elif table == 'toms': file_in = self.destination + '/WORK/all_toms_sorted' indices = [('philo_type',), ('philo_id',)] + self.metadata_fields depth = 7 compressed = False post_filter = make_sql_table(table, file_in, gz=compressed, indices=indices, depth=depth) self.post_filters.insert(0, post_filter)