Пример #1
0
 def setup_sql_load(self):
     """Setup SQL DB creation"""
     for table in self.tables:
         if table == "words":
             file_in = self.destination + "/WORK/all_words_ordered"
             indices = [("philo_name",), ("philo_id",), ("parent",), ("start_byte",), ("end_byte",)]
             depth = 7
         elif table == "pages":
             file_in = self.destination + "/WORK/all_pages"
             indices = [("philo_id",)]
             depth = 9
         elif table == "toms":
             file_in = self.destination + "/WORK/all_toms_sorted"
             indices = [("philo_type",), ("philo_id",), ("img",)] + self.metadata_fields
             depth = 7
         elif table == "refs":
             file_in = self.destination + "/WORK/all_refs"
             indices = [("parent",), ("target",), ("type",)]
             depth = 9
         elif table == "graphics":
             file_in = self.destination + "/WORK/all_graphics"
             indices = [("parent",), ("philo_id",)]
             depth = 9
         elif table == "lines":
             file_in = self.destination + "/WORK/all_lines"
             indices = [("doc_id", "start_byte", "end_byte")]
             depth = 9
         post_filter = make_sql_table(table, file_in, indices=indices, depth=depth)
         self.post_filters.insert(0, post_filter)
Пример #2
0
 def setup_sql_load(self):
     for table in self.tables:
         if table == 'words':
             file_in = self.destination + '/WORK/all_words_ordered'
             indices = [("philo_name", ), ('philo_id', ), ('parent', ), ('start_byte', ), ('end_byte', )]
             depth = 7
         elif table == 'pages':
             file_in = self.destination + '/WORK/all_pages'
             indices = [("philo_id", )]
             depth = 9
         elif table == 'toms':
             file_in = self.destination + '/WORK/all_toms_sorted'
             indices = [('philo_type', ), ('philo_id', ), ('img', )] + self.metadata_fields
             depth = 7
         elif table == "refs":
             file_in = self.destination + '/WORK/all_refs'
             indices = [("parent", ), ("target", ), ("type", )]
             depth = 9
         elif table == "graphics":
             file_in = self.destination + '/WORK/all_graphics'
             indices = [("parent", ), ("philo_id", )]
             depth = 9
         elif table == "lines":
             file_in = self.destination + '/WORK/all_lines'
             indices = [("doc_id", "start_byte", "end_byte")]
             depth = 9
         post_filter = make_sql_table(table, file_in, indices=indices, depth=depth)
         self.post_filters.insert(0, post_filter)
Пример #3
0
 def setup_sql_load(self):
     for table in self.tables:
         if table == 'words':
             file_in = self.destination + '/WORK/all_words_ordered'
             indices = [("philo_name", ), ('philo_id', ), ('parent', ),
                        ('start_byte', ), ('end_byte', )]
             depth = 7
         elif table == 'pages':
             file_in = self.destination + '/WORK/all_pages'
             indices = [("philo_id", )]
             depth = 9
         elif table == 'toms':
             file_in = self.destination + '/WORK/all_toms_sorted'
             indices = [('philo_type', ), ('philo_id', ),
                        ('img', )] + self.metadata_fields
             depth = 7
         elif table == "refs":
             file_in = self.destination + '/WORK/all_refs'
             indices = [("parent", ), ("target", ), ("type", )]
             depth = 9
         elif table == "graphics":
             file_in = self.destination + '/WORK/all_graphics'
             indices = [("parent", ), ("philo_id", )]
             depth = 9
         elif table == "lines":
             file_in = self.destination + '/WORK/all_lines'
             indices = [("doc_id", "start_byte", "end_byte")]
             depth = 9
         post_filter = make_sql_table(table,
                                      file_in,
                                      indices=indices,
                                      depth=depth)
         self.post_filters.insert(0, post_filter)
Пример #4
0
 def setup_sql_load(self):
     for table in self.tables:
         if table == 'words':
             file_in = self.destination + '/WORK/all_words_sorted.gz'
             indices = [("philo_name",
                         "%s_ancestor" % self.default_object_level),
                        ('philo_id', )]
             depth = 7
             compressed = True
         elif table == 'pages':
             file_in = self.destination + '/WORK/all_pages'
             indices = [("philo_id", )]
             depth = 9
             compressed = False
         elif table == 'toms':
             file_in = self.destination + '/WORK/all_toms_sorted'
             indices = [('philo_type', ),
                        ('philo_id', )] + self.metadata_fields
             depth = 7
             compressed = False
         post_filter = make_sql_table(table,
                                      file_in,
                                      gzip=compressed,
                                      indices=indices,
                                      depth=depth)
         self.post_filters.insert(0, post_filter)
Пример #5
0
 def setup_sql_load(self):
     for table in self.tables:
         if table == 'words':
             file_in = self.destination + '/WORK/all_words_sorted.gz'
             indices = [("philo_name", "%s_ancestor" % self.default_object_level), ('philo_id',)]
             depth = 7
             compressed = True
         elif table == 'pages':
             file_in = self.destination + '/WORK/all_pages'                
             indices = [("philo_id",)]
             depth = 9
             compressed = False
         elif table == 'toms':
             file_in = self.destination + '/WORK/all_toms_sorted'
             indices = [('philo_type',), ('philo_id',)] + self.metadata_fields
             depth = 7
             compressed = False
         post_filter = make_sql_table(table, file_in, gzip=compressed, indices=indices, depth=depth)
         self.post_filters.insert(0, post_filter)
Пример #6
0
 def setup_sql_load(self):
     for table in self.tables:
         if table == 'words':
             file_in = self.destination + '/WORK/all_words_ordered'
             indices = [("philo_name",), ('philo_id',), ('parent',), ('byte_start',), ('byte_end',)]
             depth = 7
             compressed = False
         elif table == 'pages':
             file_in = self.destination + '/WORK/all_pages'
             indices = [("philo_id",)]
             depth = 9
             compressed = False
         elif table == 'toms':
             file_in = self.destination + '/WORK/all_toms_sorted'
             indices = [('philo_type',), ('philo_id',)] + self.metadata_fields
             depth = 7
             compressed = False
         post_filter = make_sql_table(table, file_in, gz=compressed, indices=indices, depth=depth)
         self.post_filters.insert(0, post_filter)