def _save_or_replace(self, id_, obj): # build row to insert/update fields = {'id': id_} with closing(StringIO()) as dstdata: make_pickler(self, dstdata, obj).dump(obj) fields['data'] = dstdata.getvalue() try: fields['state'] = obj.execution.state except AttributeError: # If we cannot determine the state of a task, consider it UNKNOWN. fields['state'] = Run.State.UNKNOWN # insert into db for column in self.extra_fields: try: fields[column] = self.extra_fields[column](obj) gc3libs.log.debug( "Writing value '%s' in column '%s' for object '%s'", fields[column], column, obj) except Exception as ex: gc3libs.log.warning( "Error saving DB column '%s' of object '%s': %s: %s", column, obj, ex.__class__.__name__, str(ex)) with closing(self._engine.connect()) as conn: q = sql.select([self._tables.c.id]).where(self._tables.c.id == id_) r = conn.execute(q) if not r.fetchone(): # It's an insert q = self._tables.insert().values(**fields) conn.execute(q) else: # it's an update q = self._tables.update().where( self._tables.c.id == id_).values(**fields) conn.execute(q) obj.persistent_id = id_ if hasattr(obj, 'changed'): obj.changed = False # return id return obj.persistent_id
def _save_or_replace(self, id_, obj): """ Save `obj` into file identified by `id_`; if no such destination file exists, create it. Ensure that the destination file is kept intact in case dumping `obj` fails. """ filename = os.path.join(self._directory, id_) # gc3libs.log.debug("Storing job '%s' into file '%s'", obj, filename) if not os.path.exists(self._directory): try: os.makedirs(self._directory) except Exception as ex: # raise same exception but add context message gc3libs.log.error("Could not create jobs directory '%s': %s" % (self._directory, str(ex))) raise backup = None if os.path.exists(filename): backup = filename + '.OLD' os.rename(filename, backup) # TODO: this should become `with tgt = ...:` as soon as we # stop supporting Python 2.4 tgt = None try: tgt = open(filename, 'w+b') pickler = make_pickler(self, tgt, obj) pickler.dump(obj) if hasattr(obj, 'changed'): obj.changed = False tgt.close() try: os.remove(backup) except: pass # ignore errors except Exception as ex: gc3libs.log.error("Error saving job '%s' to file '%s': %s: %s", obj, filename, ex.__class__.__name__, ex) if tgt is not None: try: tgt.close() except: pass # ignore errors if backup is not None: try: os.rename(backup, filename) except: pass # ignore errors raise
def _save_or_replace(self, id_, obj): """ Save `obj` into file identified by `id_`; if no such destination file exists, create it. Ensure that the destination file is kept intact in case dumping `obj` fails. """ filename = os.path.join(self._directory, id_) # gc3libs.log.debug("Storing job '%s' into file '%s'", obj, filename) if not os.path.exists(self._directory): try: os.makedirs(self._directory) except Exception as ex: # raise same exception but add context message gc3libs.log.error("Could not create jobs directory '%s': %s" % (self._directory, str(ex))) raise backup = None if os.path.exists(filename): backup = filename + '.OLD' os.rename(filename, backup) with open(filename, 'w+b') as tgt: try: pickler = make_pickler(self, tgt, obj) pickler.dump(obj) except Exception as err: gc3libs.log.error( "Error saving task '%s' to file '%s': %s: %s", obj, filename, err.__class__.__name__, err) # move backup file back in place if backup is not None: try: os.rename(backup, filename) except: pass # ignore errors raise if hasattr(obj, 'changed'): obj.changed = False # remove backup file, if exists try: os.remove(backup) except: pass # ignore errors # update cache if id_ in self._loaded: old = self._loaded[str(id_)] if old is not obj: self._loaded[str(id_)] = obj
def _save_or_replace(self, id_, obj): fields = {'id': id_} dstdata = StringIO.StringIO() pickler = make_pickler(self, dstdata, obj) pickler.dump(obj) fields['data'] = dstdata.getvalue() try: fields['state'] = obj.execution.state except AttributeError: # If we cannot determine the state of a task, consider it UNKNOWN. fields['state'] = Run.State.UNKNOWN # insert into db for column in self.extra_fields: try: fields[column] = self.extra_fields[column](obj) gc3libs.log.debug("Writing value '%s' in column '%s' for object '%s'", fields[column], column, obj) except Exception, ex: gc3libs.log.warning("Error saving DB column '%s' of object '%s': %s: %s", column, obj, ex.__class__.__name__, str(ex))
def _save_or_replace(self, id_, obj): # if __debug__: # global _lvl # _lvl += '>' # gc3libs.log.debug("%s Saving %r@%x as %s ...", _lvl, obj, id(obj), id_) # build row to insert/update fields = {'id': id_} with closing(BytesIO()) as dstdata: make_pickler(self, dstdata, obj).dump(obj) fields['data'] = dstdata.getvalue() try: fields['state'] = obj.execution.state except AttributeError: # If we cannot determine the state of a task, consider it UNKNOWN. fields['state'] = Run.State.UNKNOWN # insert into db for column in self.extra_fields: try: fields[column] = self.extra_fields[column](obj) except Exception as ex: gc3libs.log.warning( "Error saving DB column '%s' of object '%s': %s: %s", column, obj, ex.__class__.__name__, str(ex)) if __debug__: for column in fields: if column == 'data': continue gc3libs.log.debug( "Writing value '%s' in column '%s' for object '%s'", fields[column], column, obj) q = sql.select([self._tables.c.id]).where(self._tables.c.id == id_) with self._engine.begin() as conn: r = conn.execute(q) if not r.fetchone(): # It's an insert q = self._tables.insert().values(**fields) else: # it's an update q = self._tables.update().where( self._tables.c.id == id_).values(**fields) conn.execute(q) obj.persistent_id = id_ if hasattr(obj, 'changed'): obj.changed = False # update cache if str(id_) in self._loaded: old = self._loaded[str(id_)] if old is not obj: self._loaded[str(id_)] = obj # if __debug__: # gc3libs.log.debug( # "%s Overwriting object %s %r@%x with %r@%x", # _lvl, id_, old, id(old), obj, id(obj)) # from traceback import format_stack # gc3libs.log.debug("Traceback:\n%s", ''.join(format_stack())) # if __debug__: # gc3libs.log.debug("%s Done saving %r@%x as %s ...", _lvl, obj, id(obj), id_) # if _lvl: # _lvl = _lvl[:-1] # return id return id_
# raise same exception but add context message gc3libs.log.error("Could not create jobs directory '%s': %s" % (self._directory, str(ex))) raise backup = None if os.path.exists(filename): backup = filename + '.OLD' os.rename(filename, backup) # TODO: this should become `with tgt = ...:` as soon as we # stop supporting Python 2.4 tgt = None try: tgt = open(filename, 'w+b') pickler = make_pickler(self, tgt, obj) pickler.dump(obj) if hasattr(obj, 'changed'): obj.changed = False tgt.close() try: os.remove(backup) except: pass # ignore errors except Exception, ex: gc3libs.log.error("Error saving job '%s' to file '%s': %s: %s", obj, filename, ex.__class__.__name__, ex) if tgt is not None: try: tgt.close() except: