def solveDependencyRecursive(table): """ Gets all tables on which the given table depends and that need to be rebuilt. Also will mark tables skipped which won't be rebuilt. Uses parent's variables to store data. :type table: str :param table: table name for which to solve dependencies """ if table in tableNames: # don't add dependant tables if they are given explicitly return if self.db.hasTable(table): skippedTables.add(table) return dependedTablesNames.add(table) # add dependent tables if needed (recursively) if table not in self._tableBuilderLookup: # either we have no builder or the builder was removed in # favour of another builder that shares at least one table # with the removed one raise exception.UnsupportedError("Table '%s'" % table \ + " not provided, might be related to conflicting " \ + "builders") builderClass = self._tableBuilderLookup[table] for dependantTable in builderClass.DEPENDS: solveDependencyRecursive(dependantTable)
def getTableBuilder(self, tableName): """ Gets the :class:`~cjklib.build.builder.TableBuilder` used by this instance of the database builder to build the given table. :type tableName: str :param tableName: name of table :rtype: classobj :return: :class:`~cjklib.build.builder.TableBuilder` used to build the given table by this build instance. :raise UnsupportedError: if an unsupported table is given. """ if tableName not in self._tableBuilderLookup: # either we have no builder or the builder was removed in favour # of another builder that shares at least one table with the # removed one raise exception.UnsupportedError("table '%s'" + tableName \ + " not provided, might be related to conflicting " \ + "builders") return self._tableBuilderLookup[tableName]
def remove(self, tables): """ Removes the given tables from the main database. :type tables: list :param tables: list of tables to remove :raise UnsupportedError: if an unsupported table is given. :rtype: list :return: names of deleted tables, might be smaller than the actual list """ if type(tables) != type([]): tables = [tables] tableBuilderClasses = [] for table in set(tables): if table not in self._tableBuilderLookup: raise exception.UnsupportedError("Table '%s' not provided" % table) tableBuilderClasses.append(self._tableBuilderLookup[table]) removed = [] for builder in tableBuilderClasses: if self.db.mainHasTable(builder.PROVIDES): if not self.quiet: warn("Removing previously built table '%s'" % builder.PROVIDES) # get specific options given to the DatabaseBuilder options = self.getBuilderOptions(builder, ignoreUnknown=True) options['dbConnectInst'] = self.db instance = builder(**options) instance.remove() removed.append(builder.PROVIDES) # remove old metadata if builder.PROVIDES in self.db.tables: del self.db.tables[builder.PROVIDES] return removed
def getClassesInBuildOrder(self, tableNames): """ Gets the build order for the given table names. :type tableNames: list of str :param tableNames: list of names of tables to build :rtype: list of classobj :return: :class:`~cjklib.build.builder.TableBuilder` classes in build order :raise UnsupportedError: if an unsupported table is given. """ # get dependencies and save order tableBuilderClasses = [] for table in set(tableNames): if table not in self._tableBuilderLookup: # either we have no builder or the builder was removed in favour # of another builder that shares at least one table with the # removed one raise exception.UnsupportedError("table '" + table \ + "' not provided, might be related to conflicting " \ + "builders") tableBuilderClasses.append(self._tableBuilderLookup[table]) return self.getBuildDependencyOrder(tableBuilderClasses)
def build(self, tables): """ Builds the given tables. :type tables: list :param tables: list of tables to build :raise IOError: if a table builder fails to read its data; only if :attr:`~cjklib.build.DatabaseBuilder.noFail` is set to ``False`` """ if type(tables) != type([]): tables = [tables] if not self.quiet: warn("Building database '%s'" % self.db.databaseUrl) if self.db.attached: warn("Reading from additional databases '%s'" % "', '".join(list(self.db.attached.keys()))) # remove tables that don't need to be rebuilt filteredTables = [] for table in tables: if table not in self._tableBuilderLookup: raise exception.UnsupportedError("Table '%s' not provided" \ % table) if self.needsRebuild(table): filteredTables.append(table) else: if not self.quiet: warn("Skipping table '%s' because it already exists" \ % table) tables = filteredTables # get depending tables that need to be updated when dependencies change dependingTables = [] if self.rebuildDepending: # report tables that should be updated but lie outside our scope if not self.quiet: externalDependingTables \ = self.getExternalRebuiltDependingTables(tables) if externalDependingTables: warn("Ignoring tables with dependencies updated" + " but belonging to attached databases: '" \ + "', '".join(externalDependingTables) + "'") dependingTables = self.getRebuiltDependingTables(tables) if dependingTables: if not self.quiet: warn("Tables rebuilt because of dependencies updated: '" \ + "', '".join(dependingTables) + "'") tables.extend(dependingTables) # get table list according to dependencies buildDependentTables = self.getBuildDependentTables(tables) buildTables = set(tables) | buildDependentTables # get build order and remove tables we don't need to build builderClasses = self.getClassesInBuildOrder(buildTables) # build tables if not self.quiet and self.rebuildExisting: warn("Rebuilding tables and overwriting old ones...") builderClasses.reverse() self._instancesUnrequestedTable = set() while builderClasses: builder = builderClasses.pop() transaction = self.db.connection.begin() try: # get specific options given to the DatabaseBuilder options = self.getBuilderOptions(builder, ignoreUnknown=True) options['dbConnectInst'] = self.db instance = builder(**options) # mark tables as deletable if its only provided because of # dependencies and the table doesn't exists yet if builder.PROVIDES in buildDependentTables \ and not self.db.mainHasTable(builder.PROVIDES): self._instancesUnrequestedTable.add(instance) if self.db.mainHasTable(builder.PROVIDES): # will only remove the table if found in the main database if not self.quiet: warn("Removing previously built table '%s'" % builder.PROVIDES) instance.remove() if not self.quiet: warn("Building table '%s' with builder '%s'..." % (builder.PROVIDES, builder.__name__)) # remove old metadata if builder.PROVIDES in self.db.tables: del self.db.tables[builder.PROVIDES] instance.build() transaction.commit() except IOError as e: transaction.rollback() # data not available, can't build table if self.noFail: if not self.quiet: warn("Building table '%s' failed: '%s', skipping" \ % (builder.PROVIDES, str(e))) dependingTables = [builder.PROVIDES] remainingBuilderClasses = [] for clss in builderClasses: if set(clss.DEPENDS) & set(dependingTables): # this class depends on one being removed dependingTables.append(clss.PROVIDES) else: remainingBuilderClasses.append(clss) if not self.quiet and len(dependingTables) > 1: warn("Ignoring depending table(s) '%s'" \ % "', '".join(dependingTables[1:])) builderClasses = remainingBuilderClasses else: if not self.quiet: warn("Error") self.clearTemporary() raise except Exception as e: transaction.rollback() if not self.quiet: warn("Error") self.clearTemporary() raise self.clearTemporary()