def solveDependencyRecursive(table):
"""
Gets all tables on which the given table depends and that need to be
rebuilt. Also will mark tables skipped which won't be rebuilt.
Uses parent's variables to store data.
:type table: str
:param table: table name for which to solve dependencies
"""
if table in tableNames:
# don't add dependant tables if they are given explicitly
return
if self.db.hasTable(table):
skippedTables.add(table)
return
dependedTablesNames.add(table)
# add dependent tables if needed (recursively)
if table not in self._tableBuilderLookup:
# either we have no builder or the builder was removed in
# favour of another builder that shares at least one table
# with the removed one
raise exception.UnsupportedError("Table '%s'" % table \
+ " not provided, might be related to conflicting " \
+ "builders")
builderClass = self._tableBuilderLookup[table]
for dependantTable in builderClass.DEPENDS:
solveDependencyRecursive(dependantTable)
def getTableBuilder(self, tableName):
"""
Gets the :class:`~cjklib.build.builder.TableBuilder` used by
this instance of the database builder to build the given table.
:type tableName: str
:param tableName: name of table
:rtype: classobj
:return: :class:`~cjklib.build.builder.TableBuilder` used to
build the given table by this build instance.
:raise UnsupportedError: if an unsupported table is given.
"""
if tableName not in self._tableBuilderLookup:
# either we have no builder or the builder was removed in favour
# of another builder that shares at least one table with the
# removed one
raise exception.UnsupportedError("table '%s'" + tableName \
+ " not provided, might be related to conflicting " \
+ "builders")
return self._tableBuilderLookup[tableName]
def remove(self, tables):
"""
Removes the given tables from the main database.
:type tables: list
:param tables: list of tables to remove
:raise UnsupportedError: if an unsupported table is given.
:rtype: list
:return: names of deleted tables, might be smaller than the actual list
"""
if type(tables) != type([]):
tables = [tables]
tableBuilderClasses = []
for table in set(tables):
if table not in self._tableBuilderLookup:
raise exception.UnsupportedError("Table '%s' not provided"
% table)
tableBuilderClasses.append(self._tableBuilderLookup[table])
removed = []
for builder in tableBuilderClasses:
if self.db.mainHasTable(builder.PROVIDES):
if not self.quiet:
warn("Removing previously built table '%s'"
% builder.PROVIDES)
# get specific options given to the DatabaseBuilder
options = self.getBuilderOptions(builder, ignoreUnknown=True)
options['dbConnectInst'] = self.db
instance = builder(**options)
instance.remove()
removed.append(builder.PROVIDES)
# remove old metadata
if builder.PROVIDES in self.db.tables:
del self.db.tables[builder.PROVIDES]
return removed
def getClassesInBuildOrder(self, tableNames):
"""
Gets the build order for the given table names.
:type tableNames: list of str
:param tableNames: list of names of tables to build
:rtype: list of classobj
:return: :class:`~cjklib.build.builder.TableBuilder` classes in build
order
:raise UnsupportedError: if an unsupported table is given.
"""
# get dependencies and save order
tableBuilderClasses = []
for table in set(tableNames):
if table not in self._tableBuilderLookup:
# either we have no builder or the builder was removed in favour
# of another builder that shares at least one table with the
# removed one
raise exception.UnsupportedError("table '" + table \
+ "' not provided, might be related to conflicting " \
+ "builders")
tableBuilderClasses.append(self._tableBuilderLookup[table])
return self.getBuildDependencyOrder(tableBuilderClasses)
def build(self, tables):
"""
Builds the given tables.
:type tables: list
:param tables: list of tables to build
:raise IOError: if a table builder fails to read its data; only if
:attr:`~cjklib.build.DatabaseBuilder.noFail` is set to ``False``
"""
if type(tables) != type([]):
tables = [tables]
if not self.quiet:
warn("Building database '%s'" % self.db.databaseUrl)
if self.db.attached:
warn("Reading from additional databases '%s'"
% "', '".join(list(self.db.attached.keys())))
# remove tables that don't need to be rebuilt
filteredTables = []
for table in tables:
if table not in self._tableBuilderLookup:
raise exception.UnsupportedError("Table '%s' not provided" \
% table)
if self.needsRebuild(table):
filteredTables.append(table)
else:
if not self.quiet:
warn("Skipping table '%s' because it already exists" \
% table)
tables = filteredTables
# get depending tables that need to be updated when dependencies change
dependingTables = []
if self.rebuildDepending:
# report tables that should be updated but lie outside our scope
if not self.quiet:
externalDependingTables \
= self.getExternalRebuiltDependingTables(tables)
if externalDependingTables:
warn("Ignoring tables with dependencies updated"
+ " but belonging to attached databases: '" \
+ "', '".join(externalDependingTables) + "'")
dependingTables = self.getRebuiltDependingTables(tables)
if dependingTables:
if not self.quiet:
warn("Tables rebuilt because of dependencies updated: '" \
+ "', '".join(dependingTables) + "'")
tables.extend(dependingTables)
# get table list according to dependencies
buildDependentTables = self.getBuildDependentTables(tables)
buildTables = set(tables) | buildDependentTables
# get build order and remove tables we don't need to build
builderClasses = self.getClassesInBuildOrder(buildTables)
# build tables
if not self.quiet and self.rebuildExisting:
warn("Rebuilding tables and overwriting old ones...")
builderClasses.reverse()
self._instancesUnrequestedTable = set()
while builderClasses:
builder = builderClasses.pop()
transaction = self.db.connection.begin()
try:
# get specific options given to the DatabaseBuilder
options = self.getBuilderOptions(builder, ignoreUnknown=True)
options['dbConnectInst'] = self.db
instance = builder(**options)
# mark tables as deletable if its only provided because of
# dependencies and the table doesn't exists yet
if builder.PROVIDES in buildDependentTables \
and not self.db.mainHasTable(builder.PROVIDES):
self._instancesUnrequestedTable.add(instance)
if self.db.mainHasTable(builder.PROVIDES):
# will only remove the table if found in the main database
if not self.quiet:
warn("Removing previously built table '%s'"
% builder.PROVIDES)
instance.remove()
if not self.quiet:
warn("Building table '%s' with builder '%s'..."
% (builder.PROVIDES, builder.__name__))
# remove old metadata
if builder.PROVIDES in self.db.tables:
del self.db.tables[builder.PROVIDES]
instance.build()
transaction.commit()
except IOError as e:
transaction.rollback()
# data not available, can't build table
if self.noFail:
if not self.quiet:
warn("Building table '%s' failed: '%s', skipping" \
% (builder.PROVIDES, str(e)))
dependingTables = [builder.PROVIDES]
remainingBuilderClasses = []
for clss in builderClasses:
if set(clss.DEPENDS) & set(dependingTables):
# this class depends on one being removed
dependingTables.append(clss.PROVIDES)
else:
remainingBuilderClasses.append(clss)
if not self.quiet and len(dependingTables) > 1:
warn("Ignoring depending table(s) '%s'" \
% "', '".join(dependingTables[1:]))
builderClasses = remainingBuilderClasses
else:
if not self.quiet: warn("Error")
self.clearTemporary()
raise
except Exception as e:
transaction.rollback()
if not self.quiet: warn("Error")
self.clearTemporary()
raise
self.clearTemporary()