def addOrModifyFileData(self, statusKey=None, statusValue=None, filename=None, externalSession=None, errorMessage=None, errorKey=None, fileData=None, extraData=None): try: if (externalSession is None): session = DBConnector().getNewSession() else: session = externalSession if (fileData is None): fileData = FileDataDao.getFileData(filename, session) setattr(fileData, statusKey, statusValue) self.setErrorMessage(errorMessage=errorMessage, errorKey=errorKey, extraData=extraData, fileData=fileData) Dao().addObject(objectToAdd=fileData, session=session, doCommit=True) if (externalSession is None): session.close() return fileData except Exception as e: logging.getLogger(Constant.LOGGER_GENERAL).exception(e) raise e
def addObject(self, objectToAdd, session=None, doCommit=False, doFlush=False): if (session is None): internalSession = DBConnector().getNewSession() else: internalSession = session internalSession.add(objectToAdd) if (doCommit): internalSession.commit() elif (doFlush): internalSession.flush() if (session is None): internalSession.close() logging.getLogger(Constant.LOGGER_ADDTODB).debug("Added to DB " + str(objectToAdd))
class AbstractImporter(object): cacheDict = {} logger = None def __init__(self, errorKey, filename, replace, previousStatus=None, actualStatus=None, isNullPool=False): self.initLogger() self.errorKey = errorKey self.filename = filename self.fileDataDao = FileDataDao() self.session = DBConnector(isNullPool=isNullPool).getNewSession() self.replace = replace self.previousStatus = previousStatus self.actualStatus = actualStatus self.fileData = FileDataDao.getFileData(self.filename, self.session) if (not AbstractImporter.cacheDict): self.initCache() def doImport(self): try: logging.info("START") time1 = datetime.now() if (self.skipOrProcess()): self.addOrModifyInit() if (self.replace): self.deleteImportedObject() self.logger.debug("**********START - Processing filename " + self.filename) voList = self.doImport2() persistentList = self.getPersistentList(voList) Dao().addObjectList(persistentList, self.session) self.setFileDataStatus(voList) if (voList is None): self.logger.info("***********FINISH AT " + str(datetime.now() - time1) + " " + self.filename) else: self.logger.info("***********FINISH AT " + str(datetime.now() - time1) + " " + self.filename + " objects added " + str(len(voList))) else: self.logger.info("Skipped " + self.filename) except (CustomException) as e: self.logger.error(self.filename + " " + e.status + " " + str(e)) self.addOrModifyFDError1(e) except MemoryError as e: self.logger.error(self.filename + " " + str(e)) FileDataDao().addOrModifyFileData( statusKey=self.actualStatus, statusValue=Constant.STATUS_ERROR, filename=self.filename, errorMessage='MemoryError', errorKey=self.errorKey) except ExpatError as e: self.logger.error(self.filename + " " + str(e)) self.session.rollback() self.session.close() self.session = DBConnector().getNewSession() self.addOrModifyFDError2('not well-formed') except Exception as e: self.logger.error(self.filename + " " + str(e)) self.session.rollback() self.session.close() self.session = DBConnector().getNewSession() self.addOrModifyFDError2(str(e)[0:149]) finally: self.session.commit() self.session.close() @abstractmethod def setFileDataStatus(self, voList): if (voList is not None): missingObjects = self.getMissingObjects() if (len(voList) == 0): setattr(self.fileData, self.actualStatus, Constant.STATUS_NO_DATA) elif (len(missingObjects) > 0): setattr(self.fileData, self.actualStatus, Constant.STATUS_WARNING) FileDataDao().setErrorMessage( errorMessage=str(missingObjects)[0:149], errorKey=self.errorKey, fileData=self.fileData) else: setattr(self.fileData, self.actualStatus, Constant.STATUS_OK) else: setattr(self.fileData, self.actualStatus, Constant.STATUS_OK) Dao().addObject(objectToAdd=self.fileData, session=self.session, doCommit=True) @abstractmethod def addOrModifyInit(self): self.fileDataDao.addOrModifyFileData(statusKey=self.actualStatus, statusValue=Constant.STATUS_INIT, filename=self.filename, errorKey=self.errorKey, externalSession=self.session) @abstractmethod def addOrModifyFDError1(self, e): self.fileDataDao.addOrModifyFileData(statusKey=self.actualStatus, statusValue=e.status, filename=self.filename, errorMessage=e.message, errorKey=self.errorKey, externalSession=self.session, extraData=e.extraData) @abstractmethod def addOrModifyFDError2(self, errorMessage): FileDataDao().addOrModifyFileData(statusKey=self.actualStatus, statusValue=Constant.STATUS_ERROR, filename=self.filename, errorMessage=errorMessage, errorKey=self.errorKey) @abstractmethod def doImport2(self): pass @abstractmethod def getMissingObjects(self): return [] @abstractmethod def skipOrProcess(self): #self.previousStatus is None for fileStatus if (self.previousStatus is None or (getattr(self.fileData, self.previousStatus) in [ Constant.STATUS_OK, Constant.STATUS_WARNING, Constant.STATUS_NO_DATA ])): if (getattr(self.fileData, self.actualStatus) != Constant.STATUS_OK or self.replace == True): return True else: self.logger.info("File Data Skipped " + self.filename) return False else: self.logger.info("File Data Skipped " + self.filename) return False @abstractmethod def getPersistent(self, vo): pass @abstractmethod def getPersistentList(self, voList): if (voList is not None): persistentList = [] for vo in voList: persistentList.append(self.getPersistent(vo)) return persistentList else: return [] @abstractmethod def initCache(self): pass @abstractmethod def deleteImportedObject(self): pass @abstractmethod def initLogger(self): if (AbstractImporter.logger is None or self.__class__.__name__ != AbstractImporter.logger.name): AbstractImporter.logger = createLog(self.__class__.__name__, logging.INFO) def doCommit(self): self.session.commit() def addOrModifyFDPending(self): setattr(self.fileData, self.actualStatus, Constant.STATUS_PENDING) Dao().addObject(objectToAdd=self.fileData, session=self.session, doCommit=True)