def GetOrAdd(session: sql.orm.session.Session, model, defaults: typing.Dict[str, object] = None, **kwargs): """Instantiate a mapped database object. If the object is not in the database, add it. Note that no change is written to disk until commit() is called on the session. Args: session: The database session. model: The database table class. defaults: Default values for mapped objects. kwargs: The values for the table row. Returns: An instance of the model class, with the values specified. """ instance = session.query(model).filter_by(**kwargs).first() if not instance: params = { k: v for k, v in kwargs.items() if not isinstance(v, sql.sql.expression.ClauseElement) } params.update(defaults or {}) instance = model(**params) session.add(instance) logging.Log(logging.GetCallingModuleName(), 5, 'New record: %s(%s)', model.__name__, params) return instance
def Flush(self) -> None: """Commit all buffered mapped objects to database.""" failures = ResilientAddManyAndCommit(self._db, self._to_commit) if len(failures): logging.Log(logging.GetCallingModuleName(), 1, 'BufferedDatabaseWriter failed to commit %d objects', len(failures)) self._to_commit = [] self._last_commit = time.time()
def Log(level: int, msg, *args, **kwargs): """Logs a message at the given level. Per-module verbose level. The argument has to contain a comma-separated list of <module name>=<log level>. <module name> is a glob pattern (e.g., " "gfs* for all modules whose name starts with \"gfs\"), matched against the " "filename base (that is, name ignoring .py). <log level> overrides any " "value given by --v." """ calling_module = logging.GetCallingModuleName() logging.Log(calling_module, level, msg, *args, **kwargs)
def ResilientAddManyAndCommit(db: Database, mapped: typing.Iterable[Base]): """Attempt to commit all mapped objects and return those that fail. This method creates a session and commits the given mapped objects. In case of error, this method will recurse up to O(log(n)) times, committing as many objects that can be as possible. Args: db: The database to add the objects to. mapped: A sequence of objects to commit. Returns: Any items in `mapped` which could not be committed, if any. Relative order of items is preserved. """ failures = [] if not mapped: return failures mapped = list(mapped) try: with db.Session(commit=True) as session: session.add_all(mapped) except sql.exc.SQLAlchemyError as e: logging.Log( logging.GetCallingModuleName(), 1, 'Caught error while committing %d mapped objects: %s', len(mapped), e, ) # Divide and conquer. If we're committing only a single object, then a # failure to commit it means that we can do nothing other than return it. # Else, divide the mapped objects in half and attempt to commit as many of # them as possible. if len(mapped) == 1: return mapped else: mid = int(len(mapped) / 2) left = mapped[:mid] right = mapped[mid:] failures += ResilientAddManyAndCommit(db, left) failures += ResilientAddManyAndCommit(db, right) return failures
def LogIf(level: int, condition, msg, *args, **kwargs): if condition: calling_module = logging.GetCallingModuleName() logging.Log(calling_module, level, msg, *args, **kwargs)