def GetOrAdd(session: sql.orm.session.Session,
model,
defaults: typing.Dict[str, object] = None,
**kwargs):
"""Instantiate a mapped database object.
If the object is not in the database,
add it. Note that no change is written to disk until commit() is called on the
session.
Args:
session: The database session.
model: The database table class.
defaults: Default values for mapped objects.
kwargs: The values for the table row.
Returns:
An instance of the model class, with the values specified.
"""
instance = session.query(model).filter_by(**kwargs).first()
if not instance:
params = {
k: v
for k, v in kwargs.items()
if not isinstance(v, sql.sql.expression.ClauseElement)
}
params.update(defaults or {})
instance = model(**params)
session.add(instance)
logging.Log(logging.GetCallingModuleName(), 5, 'New record: %s(%s)',
model.__name__, params)
return instance
def Flush(self) -> None:
"""Commit all buffered mapped objects to database."""
failures = ResilientAddManyAndCommit(self._db, self._to_commit)
if len(failures):
logging.Log(logging.GetCallingModuleName(), 1,
'BufferedDatabaseWriter failed to commit %d objects',
len(failures))
self._to_commit = []
self._last_commit = time.time()
def Log(level: int, msg, *args, **kwargs):
"""Logs a message at the given level.
Per-module verbose level. The argument has to contain a comma-separated
list of <module name>=<log level>. <module name> is a glob pattern (e.g., "
"gfs* for all modules whose name starts with \"gfs\"), matched against the "
"filename base (that is, name ignoring .py). <log level> overrides any "
"value given by --v."
"""
calling_module = logging.GetCallingModuleName()
logging.Log(calling_module, level, msg, *args, **kwargs)
def ResilientAddManyAndCommit(db: Database, mapped: typing.Iterable[Base]):
"""Attempt to commit all mapped objects and return those that fail.
This method creates a session and commits the given mapped objects.
In case of error, this method will recurse up to O(log(n)) times, committing
as many objects that can be as possible.
Args:
db: The database to add the objects to.
mapped: A sequence of objects to commit.
Returns:
Any items in `mapped` which could not be committed, if any. Relative order
of items is preserved.
"""
failures = []
if not mapped:
return failures
mapped = list(mapped)
try:
with db.Session(commit=True) as session:
session.add_all(mapped)
except sql.exc.SQLAlchemyError as e:
logging.Log(
logging.GetCallingModuleName(),
1,
'Caught error while committing %d mapped objects: %s',
len(mapped),
e,
)
# Divide and conquer. If we're committing only a single object, then a
# failure to commit it means that we can do nothing other than return it.
# Else, divide the mapped objects in half and attempt to commit as many of
# them as possible.
if len(mapped) == 1:
return mapped
else:
mid = int(len(mapped) / 2)
left = mapped[:mid]
right = mapped[mid:]
failures += ResilientAddManyAndCommit(db, left)
failures += ResilientAddManyAndCommit(db, right)
return failures
def LogIf(level: int, condition, msg, *args, **kwargs):
if condition:
calling_module = logging.GetCallingModuleName()
logging.Log(calling_module, level, msg, *args, **kwargs)